File size: 1,760 Bytes
b2cb4a0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | #!/usr/bin/env python
#
# For licensing see accompanying LICENSE file.
# Copyright (c) 2025 Apple Inc. Licensed under MIT License.
#
import argparse
import sys
from pathlib import Path
from tqdm import tqdm
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from onescience.datapipes.boltz_data_pipeline.tokenize.boltz_protein import BoltzTokenizer
from onescience.datapipes.boltz_data_pipeline.types import Manifest
from onescience.datapipes.simplefold.process_structure import tokenize_structure, finalize
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Tokenize structure data.")
parser.add_argument(
"--target_dir",
type=str,
required=True,
help="Directory containing the processed structure data.",
)
parser.add_argument(
"--token_dir",
type=str,
required=True,
help="Directory to save the tokenized data.",
)
args = parser.parse_args()
target_dir = Path(args.target_dir)
manifest_path = target_dir / "manifest.json"
manifest: Manifest = Manifest.load(manifest_path)
tokenizer = BoltzTokenizer()
records = manifest.records
print(f"Number of records after filtering: {len(records)}")
save_token_dir = Path(args.token_dir) / "tokens"
save_token_record_dir = Path(args.token_dir) / "records"
save_token_dir.mkdir(parents=True, exist_ok=True)
save_token_record_dir.mkdir(parents=True, exist_ok=True)
for record in tqdm(records):
tokenize_structure(
record,
tokenizer,
target_dir,
str(save_token_dir),
save_token_record_dir,
)
finalize(Path(args.token_dir))
|