feat: binary only mode

scripts/server_data/binary_only_runners.txt

@@ -1,4 +0,0 @@
-cybergym/oss-fuzz-base-runner:latest
-cybergym/oss-fuzz-base-runner:20200102
-cybergym/oss-fuzz-base-runner:20190802
-cybergym/oss-fuzz-base-runner:20220102

scripts/server_data/download.py

@@ -46,11 +46,18 @@ if __name__ == "__main__":
 
     with open(args.tasks_file) as f:
         tasks = json.load(f)
-    tags = []
+    tags_arvo = []
+    tags_ossfuzz = []
     for task in tasks:
         if task["task_id"].split(":")[0] == "arvo":
             arvo_id = task["task_id"].split(":")[-1]
-            tags.append(f"{arvo_id}-vul")
-            tags.append(f"{arvo_id}-fix")
-    if tags:
-        pull_images("n132/arvo", tags, args.max_workers)
+            tags_arvo.append(f"{arvo_id}-vul")
+            tags_arvo.append(f"{arvo_id}-fix")
+        if task["task_id"].split(":")[0] == "ossfuzz":
+            ossfuzz_id = task["task_id"].split(":")[-1]
+            tags_ossfuzz.append(f"{ossfuzz_id}-vul")
+            tags_ossfuzz.append(f"{ossfuzz_id}-fix")
+    if tags_arvo:
+        pull_images("n132/arvo", tags_arvo, args.max_workers)
+    if tags_ossfuzz:
+        pull_images("cybergym/oss-fuzz", tags_ossfuzz, args.max_workers)

scripts/server_data/download_binary_only_runners.py

@@ -0,0 +1,36 @@
+import argparse
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import docker
+import json
+
+client = docker.from_env()
+
+
+def pull_images(repo, tags, max_workers=1):
+    def _pull(tag):
+        image = f"{repo}:{tag}"
+        print(f"Pulling {image}...")
+        try:
+            client.images.pull(repo, tag=tag)
+            print(f"Successfully pulled {image}")
+        except docker.errors.APIError as e:
+            print(f"Failed to pull {image}: {e}")
+
+    if max_workers == 1:
+        for tag in tags:
+            _pull(tag)
+        return
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(_pull, tag): tag for tag in tags}
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception as e:
+                tag = futures[future]
+                print(f"Unexpected error pulling {repo}:{tag}: {e}")
+
+
+if __name__ == "__main__":
+    pull_images("cybergym/oss-fuzz-base-runner", ["latest", "20200102", "20190802", "20220102"], 1)

scripts/server_data/download_subset.py

@@ -54,11 +54,18 @@ if __name__ == "__main__":
 
     pull_images("cybergym/oss-fuzz-base-runner", ["latest"], 1)
 
-    tags = []
+    tags_arvo = []
+    tags_ossfuzz = []
     for task_id in task_ids:
         if task_id.split(":")[0] == "arvo":
             arvo_id = task_id.split(":")[-1]
-            tags.append(f"{arvo_id}-vul")
-            tags.append(f"{arvo_id}-fix")
-    if tags:
-        pull_images("n132/arvo", tags, args.max_workers)
+            tags_arvo.append(f"{arvo_id}-vul")
+            tags_arvo.append(f"{arvo_id}-fix")
+        if task_id.split(":")[0] == "oss-fuzz":
+            ossfuzz_id = task_id.split(":")[-1]
+            tags_ossfuzz.append(f"{ossfuzz_id}-vul")
+            tags_ossfuzz.append(f"{ossfuzz_id}-fix")
+    if tags_arvo:
+        pull_images("n132/arvo", tags_arvo, args.max_workers)
+    if tags_ossfuzz:
+        pull_images("cybergym/oss-fuzz", tags_ossfuzz, args.max_workers)