mirror of
https://github.com/sunblaze-ucb/cybergym.git
synced 2026-10-12 08:42:24 +00:00
init repo
This commit is contained in:
zhun
56
scripts/server_data/download.py
Normal file
56
scripts/server_data/download.py
Normal file
@@ -0,0 +1,56 @@
|
||||
import argparse
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import docker
|
||||
import json
|
||||
|
||||
client = docker.from_env()
|
||||
|
||||
|
||||
def pull_images(repo, tags, max_workers=1):
|
||||
def _pull(tag):
|
||||
image = f"{repo}:{tag}"
|
||||
print(f"Pulling {image}...")
|
||||
try:
|
||||
client.images.pull(repo, tag=tag)
|
||||
print(f"Successfully pulled {image}")
|
||||
except docker.errors.APIError as e:
|
||||
print(f"Failed to pull {image}: {e}")
|
||||
|
||||
if max_workers == 1:
|
||||
for tag in tags:
|
||||
_pull(tag)
|
||||
return
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {executor.submit(_pull, tag): tag for tag in tags}
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
tag = futures[future]
|
||||
print(f"Unexpected error pulling {repo}:{tag}: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Download the server data.")
|
||||
parser.add_argument(
|
||||
"--tasks-file", "-f", default="tasks.json", help="Path to file containing tags (default: arvo-ids.txt)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-workers", "-w", type=int, default=1, help="Maximum number of concurrent workers (default: 1)"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
pull_images("cybergym/oss-fuzz-base-runner", ["latest"], 1)
|
||||
|
||||
with open(args.tasks_file) as f:
|
||||
tasks = json.load(f)
|
||||
tags = []
|
||||
for task in tasks:
|
||||
if task["task_id"].split(":")[0] == "arvo":
|
||||
arvo_id = task["task_id"].split(":")[-1]
|
||||
tags.append(f"{arvo_id}-vul")
|
||||
tags.append(f"{arvo_id}-fix")
|
||||
if tags:
|
||||
pull_images("n132/arvo", tags, args.max_workers)
|
||||
30
scripts/server_data/download_chunks.sh
Executable file
30
scripts/server_data/download_chunks.sh
Executable file
@@ -0,0 +1,30 @@
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-aa
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ab
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ac
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ad
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ae
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-af
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ag
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ah
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ai
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-aj
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ak
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-al
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-am
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-an
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ao
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ap
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-aq
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ar
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-as
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-at
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-au
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-av
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-aw
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ax
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ay
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-az
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-ba
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-bb
|
||||
wget https://huggingface.co/datasets/sunblaze-ucb/cybergym-server/resolve/main/cybergym-oss-fuzz-data.7z.part-bc
|
||||
cat cybergym-oss-fuzz-data.7z.part-* > cybergym-oss-fuzz-data.7z
|
||||
64
scripts/server_data/download_subset.py
Normal file
64
scripts/server_data/download_subset.py
Normal file
@@ -0,0 +1,64 @@
|
||||
import argparse
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import docker
|
||||
import json
|
||||
|
||||
client = docker.from_env()
|
||||
|
||||
task_ids = [
|
||||
"arvo:47101",
|
||||
"arvo:3938",
|
||||
"arvo:24993",
|
||||
"arvo:1065",
|
||||
"arvo:10400",
|
||||
"arvo:368",
|
||||
"oss-fuzz:42535201",
|
||||
"oss-fuzz:42535468",
|
||||
"oss-fuzz:370689421",
|
||||
"oss-fuzz:385167047",
|
||||
]
|
||||
|
||||
|
||||
def pull_images(repo, tags, max_workers=1):
|
||||
def _pull(tag):
|
||||
image = f"{repo}:{tag}"
|
||||
print(f"Pulling {image}...")
|
||||
try:
|
||||
client.images.pull(repo, tag=tag)
|
||||
print(f"Successfully pulled {image}")
|
||||
except docker.errors.APIError as e:
|
||||
print(f"Failed to pull {image}: {e}")
|
||||
|
||||
if max_workers == 1:
|
||||
for tag in tags:
|
||||
_pull(tag)
|
||||
return
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {executor.submit(_pull, tag): tag for tag in tags}
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
tag = futures[future]
|
||||
print(f"Unexpected error pulling {repo}:{tag}: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Download the server data.")
|
||||
parser.add_argument(
|
||||
"--max-workers", "-w", type=int, default=1, help="Maximum number of concurrent workers (default: 1)"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
pull_images("cybergym/oss-fuzz-base-runner", ["latest"], 1)
|
||||
|
||||
tags = []
|
||||
for task_id in task_ids:
|
||||
if task_id.split(":")[0] == "arvo":
|
||||
arvo_id = task_id.split(":")[-1]
|
||||
tags.append(f"{arvo_id}-vul")
|
||||
tags.append(f"{arvo_id}-fix")
|
||||
if tags:
|
||||
pull_images("n132/arvo", tags, args.max_workers)
|
||||
Reference in New Issue
Block a user