loop-benchmarking

generate.py (5252B)

---

 #!/usr/bin/env python3
 """Generate noise files for context_noise axis.
 
 Creates wikipedia and lorem ipsum text files at 25/50/75% of context window.
 Target sizes: 25%=200KB, 50%=400KB, 75%=600KB
 
 Usage: python3 tasks/tetris/noise/generate.py
 """
 
 import json
 import urllib.request
 import os
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 # Target sizes in characters
 SIZES = {
     "25": 200_000,
     "50": 400_000,
     "75": 600_000,
 }
 
 WIKI_ARTICLES = [
     "Roman_Empire", "French_Revolution", "World_War_II", "Ancient_Egypt",
     "Quantum_mechanics", "Theory_of_relativity", "Evolution", "DNA",
     "Amazon_rainforest", "Pacific_Ocean", "Himalayas",
     "Philosophy_of_mind", "Ethics", "Epistemology",
     "Computer_science", "Algorithm", "Machine_learning", "Cryptography",
     "Renaissance", "Industrial_Revolution", "Cold_War", "Byzantine_Empire",
     "Solar_System", "Photosynthesis", "Plate_tectonics", "Periodic_table",
     "Democracy", "Capitalism", "Feudalism", "Imperialism",
     "Mozart", "Shakespeare", "Leonardo_da_Vinci", "Isaac_Newton",
     "Olympic_Games", "Chess", "Association_football",
     "Universe", "Black_hole", "Milky_Way", "Jupiter",
 ]
 
 LOREM_BLOCK = (
     "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod "
     "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim "
     "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea "
     "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate "
     "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint "
     "occaecat cupidatat non proident, sunt in culpa qui officia deserunt "
     "mollit anim id est laborum.\n\n"
     "Sed ut perspiciatis unde omnis iste natus error sit voluptatem "
     "accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae "
     "ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt "
     "explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut "
     "odit aut fugit, sed quia consequuntur magni dolores eos qui ratione "
     "voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum "
     "quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam "
     "eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat "
     "voluptatem.\n\n"
     "At vero eos et accusamus et iusto odio dignissimos ducimus qui "
     "blanditiis praesentium voluptatum deleniti atque corrupti quos dolores "
     "et quas molestias excepturi sint occaecati cupiditate non provident, "
     "similique sunt in culpa qui officia deserunt mollitia animi, id est "
     "laborum et dolorum fuga. Et harum quidem rerum facilis est et expedita "
     "distinctio. Nam libero tempore, cum soluta nobis est eligendi optio "
     "cumque nihil impedit quo minus id quod maxime placeat facere possimus, "
     "omnis voluptas assumenda est, omnis dolor repellendus.\n\n"
 )
 
 
 def fetch_wiki_text(title):
     """Fetch plain text extract of a Wikipedia article."""
     url = (
         "https://en.wikipedia.org/w/api.php?"
         f"action=query&titles={title}&prop=extracts&explaintext=true&format=json"
     )
     try:
         req = urllib.request.Request(url, headers={"User-Agent": "LoopBench/1.0"})
         with urllib.request.urlopen(req, timeout=15) as resp:
             data = json.loads(resp.read())
             pages = data.get("query", {}).get("pages", {})
             for page in pages.values():
                 extract = page.get("extract", "")
                 if extract:
                     return extract
     except Exception as e:
         print(f"  Failed to fetch {title}: {e}")
     return ""
 
 
 def generate_wikipedia_files():
     """Generate wikipedia noise files at 25/50/75% sizes."""
     print("Fetching Wikipedia articles...")
     all_text = ""
     for title in WIKI_ARTICLES:
         if len(all_text) >= SIZES["75"]:
             break
         text = fetch_wiki_text(title)
         if text:
             all_text += f"\n\n--- {title.replace('_', ' ')} ---\n\n{text}"
             print(f"  {title}: {len(text)} chars (total: {len(all_text)})")
 
     if len(all_text) < SIZES["75"]:
         print(f"  Warning: only got {len(all_text)} chars, need {SIZES['75']}")
         # Pad by repeating
         while len(all_text) < SIZES["75"]:
             all_text += all_text[:SIZES["75"] - len(all_text)]
 
     for level, target in SIZES.items():
         path = os.path.join(SCRIPT_DIR, f"wikipedia_{level}.txt")
         content = all_text[:target]
         with open(path, "w") as f:
             f.write(content)
         print(f"  wikipedia_{level}.txt: {len(content)} chars ({len(content)//1000}KB)")
 
 
 def generate_lorem_files():
     """Generate lorem ipsum noise files at 25/50/75% sizes."""
     print("Generating lorem ipsum files...")
     for level, target in SIZES.items():
         repeats = (target // len(LOREM_BLOCK)) + 1
         content = (LOREM_BLOCK * repeats)[:target]
         path = os.path.join(SCRIPT_DIR, f"lorem_{level}.txt")
         with open(path, "w") as f:
             f.write(content)
         print(f"  lorem_{level}.txt: {len(content)} chars ({len(content)//1000}KB)")
 
 
 if __name__ == "__main__":
     generate_lorem_files()
     generate_wikipedia_files()
     print("Done.")