{ "schema_version": "2026.11", "registry": "Best AI Data Reference Registry", "domain": "bestaidata.org", "canonical_url": "https://bestaidata.org/data.json", "csv_mirror_url": "https://bestaidata.org/data.csv", "license": "https://creativecommons.org/licenses/by/4.0/", "reviewed_month": "November 2026", "primary_angle": "AI Training Dataset Modalities, Statistical Quality Standards, Copyright Provenance & Open vs. Commercial Corpora", "benchmark_matrix_columns": [ "Provider / Platform", "Primary Data Modalities", "Provenance & License Audit", "Open vs. Custom Scope", "QA / IAA Methodology" ], "benchmark_matrix": [ { "rank": 1, "name": "Surge AI", "slug": "surge-ai", "official_url": "https://www.surgehq.ai", "status": "featured_verified_entity", "neutrality_status": "100% Independent (Zero Frontier Lab Equity Ties)", "columns": { "Primary Data Modalities": "Code, STEM, Reasoning, Multimodal Text/Audio", "Provenance & License Audit": "Full Commercial IP Assignment + Clean Provenance Logs", "Open vs. Custom Scope": "Custom Proprietary Human Corpora & Golden Evals", "QA / IAA Methodology": "Programmatic IAA (alpha >= 0.85) + Expert Calibration" } }, { "rank": 2, "name": "Mercor", "slug": "mercor", "official_url": "https://mercor.com", "status": "verified_tier_partner", "neutrality_status": "100% Independent", "columns": { "Primary Data Modalities": "STEM, Software Engineering, Legal, Finance & Clinical Corpora", "Provenance & License Audit": "Work-for-Hire Commercial Assignment + Vetted Identity Lineage", "Open vs. Custom Scope": "Custom Domain-Expert Datasets & APEX Evaluation Suites", "QA / IAA Methodology": "Multi-Reviewer Consensus + Expert Benchmark Calibration" } }, { "rank": 3, "name": "micro1", "slug": "micro1", "official_url": "https://www.micro1.ai", "status": "verified_tier_partner", "neutrality_status": "100% Independent", "columns": { "Primary Data Modalities": "Code, STEM Text, Multilingual Audio & Robotics Teleoperation", "Provenance & License Audit": "Full Enterprise Copyright Transfer + Proctored Session Logs", "Open vs. Custom Scope": "Custom Human Data Engineering & Embodied AI Trajectories", "QA / IAA Methodology": "Dedicated QA Leads + Automated Outlier & IAA Screening" } }, { "rank": 4, "name": "Turing", "slug": "turing", "official_url": "https://www.turing.com", "status": "verified_tier_partner", "neutrality_status": "100% Independent", "columns": { "Primary Data Modalities": "Repository-Scale Code, Cloud Architecture & Multimodal STEM", "Provenance & License Audit": "Enterprise MSA Indemnity + Auditable Contributor Provenance", "Open vs. Custom Scope": "Proprietary Post-Training & Code-Reasoning Datasets", "QA / IAA Methodology": "Unit-Test Verification + Senior Architect Peer Review" } }, { "rank": 5, "name": "Hugging Face Datasets", "slug": "hugging-face-datasets", "official_url": "https://huggingface.co/datasets", "status": "public_benchmark", "neutrality_status": "Independent / Open Standard", "columns": { "Primary Data Modalities": "Open Pre-Training, SFT, Vision, Audio & Tabular", "Provenance & License Audit": "Per-Repo SPDX / Croissant Metadata (Varies by Uploader)", "Open vs. Custom Scope": "Open-Weight Public Hub & Community Benchmarks", "QA / IAA Methodology": "Community Cards + Downstream Cleanlab / Datatrove Filtering" } }, { "rank": 6, "name": "Scale AI", "slug": "scale-ai", "official_url": "https://scale.com", "status": "conflicted_benchmark", "neutrality_status": "Conflicted Cap Table (49% Meta Equity Stake)", "columns": { "Primary Data Modalities": "AV Sensor, Multimodal Vision & Outlier Generative Text", "Provenance & License Audit": "Commercial MSA (Requires Cap-Table Firewall Audit — 49% Meta Stake)", "Open vs. Custom Scope": "Proprietary Enterprise & Government Datasets", "QA / IAA Methodology": "Automated Consensus + Tiered Contributor Review" } }, { "rank": 7, "name": "Label Studio", "slug": "label-studio", "official_url": "https://labelstud.io", "status": "public_benchmark", "neutrality_status": "Independent / Open Standard", "columns": { "Primary Data Modalities": "Multi-Modality Text, Audio, Video, DICOM & Time-Series", "Provenance & License Audit": "Self-Hosted Customer-Owned Lineage (Apache 2.0)", "Open vs. Custom Scope": "Open-Source Annotation Infrastructure (BYO Workforce)", "QA / IAA Methodology": "Configurable Krippendorff's Alpha & Golden-Set Webhooks" } }, { "rank": 8, "name": "Cleanlab", "slug": "cleanlab", "official_url": "https://cleanlab.ai", "status": "public_benchmark", "neutrality_status": "Independent / Open Standard", "columns": { "Primary Data Modalities": "Tabular, Text, Image & Embedding Outlier Auditing", "Provenance & License Audit": "Deterministic Noise-Matrix Audit Logs", "Open vs. Custom Scope": "Open-Source Confident Learning QA Engine + Studio", "QA / IAA Methodology": "Out-of-Sample Cross-Validated Label Error Detection" } } ], "review_dossiers": [ { "index": 1, "slug": "dolma-3", "entity_name": "Dolma 3 (Ai2 Open Corpus)", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/dolma-3/", "official_url": "https://huggingface.co/allenai", "rank_label": "#1 Open Reproducible Corpus", "best_for": "Transparent LLM pre-training, ablation science & decontamination baselines", "neutrality_or_license": "ODC-By 1.0 wrapper (underlying web/PDF source terms vary)", "pricing": "Free public Parquet / JSONL.zst weights via Hugging Face & S3", "turnaround_or_scale": "5.2T+ tokens across web, olmOCR PDFs, code & math", "answer_in_brief": "Dolma 3 ranks #1 among fully reproducible academic pre-training mixtures, delivering over 5.2 trillion deduplicated tokens across web crawls, scientific PDFs, code, and mathematical reasoning shards. Released by the Allen Institute for AI under ODC-By 1.0, its curation recipes, Bloom-filter decontamination pipelines, and sub-source weights are completely transparent. It is ideal for open-science pre-training baselines, though commercial enterprises still require downstream custom human post-training data with full copyright indemnity." }, { "index": 2, "slug": "fineweb", "entity_name": "FineWeb, FineWeb-Edu & FineWeb2", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/fineweb/", "official_url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb", "rank_label": "#2 Open Web Pre-Training Suite", "best_for": "High-throughput Parquet web pre-training & multilingual base models", "neutrality_or_license": "ODC-By 1.0 (Datatrove Apache 2.0 pipeline)", "pricing": "Free public Parquet shards on Hugging Face Hub", "turnaround_or_scale": "15T English + 8T FineWeb2 multilingual tokens", "answer_in_brief": "FineWeb ranks as the most widely adopted open English and multilingual web pre-training suite, spanning 15+ trillion English tokens across 96+ Common Crawl dumps plus 8+ trillion multilingual tokens in FineWeb2. Built with Hugging Face's open Datatrove pipeline under ODC-By 1.0, its per-snapshot MinHash deduplication proved that cross-dump over-deduplication can actually degrade LLM downstream accuracy. It excels as a clean web pre-training backbone, provided teams supplement its textbook-skewed FineWeb-Edu split with authentic domain and code corpora." }, { "index": 3, "slug": "nemotron-cc", "entity_name": "Nemotron-CC (v2.1)", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/nemotron-cc/", "official_url": "https://huggingface.co/nvidia", "rank_label": "#3 Hybrid Pre-Training Corpus", "best_for": "High-token-horizon pre-training & mid-training knowledge annealing", "neutrality_or_license": "NVIDIA Open Model License / Common Crawl upstream", "pricing": "Free public release via Hugging Face / NeMo Curator", "turnaround_or_scale": "6.3T tokens (4.4T real + 1.9T synthetic rephrased)", "answer_in_brief": "Nemotron-CC ranks as the highest-density hybrid web-plus-synthetic pre-training corpus, delivering 6.3 trillion tokens (4.4T real filtered web tokens plus 1.9T synthetically rephrased tokens). Curated by NVIDIA from Common Crawl snapshots using model-ensemble classifiers rather than brittle heuristic rules, it recovers factual content from poorly formatted web pages instead of deleting 90% of crawl data. It is best suited for compute-rich pre-training teams comfortable navigating NVIDIA Open Model and Common Crawl provenance terms." }, { "index": 4, "slug": "common-crawl", "entity_name": "Common Crawl (WARC / WAT / WET)", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/common-crawl/", "official_url": "https://commoncrawl.org", "rank_label": "Upstream Web Root Archive", "best_for": "Custom HTML/DOM extraction, domain-specific vertical mining & fresh crawls", "neutrality_or_license": "501(c)(3) open archive (zero downstream copyright license granted)", "pricing": "Free on AWS S3 `s3://commoncrawl/` (requester-pays egress outside us-east-1)", "turnaround_or_scale": "2.5B–3.5B pages (~400 TiB uncompressed) per monthly crawl", "answer_in_brief": "Common Crawl is the foundational non-profit public web archive underpinning virtually every modern LLM, hosting over 250 billion crawled web pages in S3 WARC, WAT, and WET formats. While free to access under Common Crawl's Terms of Use, raw snapshots are unsuitable for direct training—over 85% of raw bytes consist of navigation chrome, SEO spam, duplicate syndicated pages, adult content, and unredacted PII. Engineering teams should ingest raw WARC archives only if operating custom petabyte-scale Datatrove or Ray extraction pipelines." }, { "index": 5, "slug": "redpajama-v2", "entity_name": "RedPajama-Data-v2", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/redpajama-v2/", "official_url": "https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2", "rank_label": "Top Signal-Annotated Raw Pool", "best_for": "Custom threshold research across EN, DE, FR, IT & ES web text", "neutrality_or_license": "Apache 2.0 pipeline / Common Crawl upstream content", "pricing": "Free public download (documents + separate quality-signal JSONs)", "turnaround_or_scale": "30T+ raw tokens (~20T deduplicated) across 84 CC dumps", "answer_in_brief": "RedPajama-Data-v2 is a 30+ trillion token uncleaned multi-lingual corpus (English, German, French, Italian, Spanish) paired with 40+ pre-computed quality annotations and MinHash signatures per document. Created by Together AI under Apache 2.0 tooling over 84 Common Crawl snapshots, it deliberately avoids baking in a single fixed filter threshold so ML researchers can design custom data mixtures via SQL/Parquet predicates. It excels for data-centric research ablations, but teams wanting turnkey training shards prefer FineWeb or Dolma 3." }, { "index": 6, "slug": "laion-bvd", "entity_name": "LAION-BVD (Big Video Dataset Ecosystem)", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/laion-bvd/", "official_url": "https://laion.ai", "rank_label": "Academic Video Metadata Index", "best_for": "Non-commercial VLM temporal alignment & academic clip-retrieval research", "neutrality_or_license": "CC-BY-4.0 index ONLY (underlying video streams un-cleared)", "pricing": "Free metadata Parquet; high compute/proxy cost to hydrate & transcode", "turnaround_or_scale": "100M+ clip metadata rows (65–75% still resolvable)", "answer_in_brief": "LAION-BVD (and adjacent open video-URL indices) distributes metadata tables of 100M+ video-caption clip pairs under CC-BY-4.0 rather than hosting raw MP4 files. While invaluable for academic spatio-temporal representation research, it is non-viable for commercial enterprise video generation or VLM production due to severe URL link rot (>25%), platform anti-scraping blocks, watermark artifacts, and zero underlying audiovisual copyright clearance. Commercial multimodal teams must license rights-cleared footage or commission custom video capture." }, { "index": 7, "slug": "nvidia-granary", "entity_name": "NVIDIA Granary (Multilingual Speech Corpus)", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/nvidia-granary/", "official_url": "https://huggingface.co/nvidia", "rank_label": "#1 Open Multilingual ASR Corpus", "best_for": "25-language European ASR transcription & speech translation pre-training", "neutrality_or_license": "CC-BY-4.0 pipeline / VoxPopuli & public-speech upstream terms", "pricing": "Free recipes & manifests via Hugging Face / NeMo", "turnaround_or_scale": "~1M hours (~650K ASR + ~350K AST translation)", "answer_in_brief": "NVIDIA Granary ranks #1 in open-ecosystem European multilingual speech scale, assembling roughly 1 million hours of Automatic Speech Recognition (ASR) and Speech-to-Text Translation (AST) audio across 25 languages. Constructed with NVIDIA's open NeMo Speech Data Processor over public civic, parliamentary (VoxPopuli), and permissive broadcast sources, it uses consensus pseudo-labeling to standardize punctuation and capitalization. It is exceptional for multilingual transcription pre-training, though full-duplex conversational voice LLMs still require studio-consented human dialogue." }, { "index": 8, "slug": "agibot-world", "entity_name": "AgiBot World (Embodied Robotics Corpus)", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/agibot-world/", "official_url": "https://huggingface.co/agibot-world", "rank_label": "#1 Scale Bimanual Robotics Set", "best_for": "Academic Vision-Language-Action (VLA) pre-training & bimanual research", "neutrality_or_license": "CC-BY-NC-SA 4.0 (Non-Commercial ShareAlike)", "pricing": "Free research download on Hugging Face; commercial use restricted", "turnaround_or_scale": "1M+ trajectories · 217 tasks · 5–8 synchronized cameras", "answer_in_brief": "AgiBot World is the largest open-access bimanual manipulation dataset released to date, capturing over 1 million real-world robot trajectories across 217 long-horizon household, retail, and industrial tasks collected by 100+ homogeneous dual-arm robots. Unlike fragmented academic multi-lab merges, its calibrated multi-camera RGB-D, fisheye wrist, joint torque, and visuotactile streams share uniform hardware kinematics. However, its CC-BY-NC-SA 4.0 research license blocks commercial deployment, forcing commercial robotics companies to commission proprietary teleoperation data." }, { "index": 9, "slug": "droid-robotics", "entity_name": "DROID (Distributed Robot Interaction Dataset)", "entity_kind": "dataset", "review_url": "https://bestaidata.org/reviews/droid-robotics/", "official_url": "https://droid-dataset.github.io", "rank_label": "#1 Commercially Clear Robotics Set", "best_for": "Commercial VLA policy pre-training & scene-generalization benchmarks", "neutrality_or_license": "CC-BY-4.0 (Commercially Cleared w/ Attribution)", "pricing": "Free public RLDS / TFRecord & LeRobot shards (~1.7 TB)", "turnaround_or_scale": "76,000 trajectories · 564 real scenes · 86 tasks", "answer_in_brief": "DROID ranks #1 among commercially permissive open robotics datasets, offering 76,000 teleoperated trajectories (350 interaction hours) collected across 564 real-world kitchens, offices, workshops, and homes under CC-BY-4.0. Coordinated by TRI, Stanford, UC Berkeley, and 13 partner institutions using identical Franka Panda 7-DoF arms with Zed stereo cameras, it provides calibrated extrinsics and natural language task instructions. It is the premier open commercial VLA bootstrap set, limited primarily by single-arm parallel-jaw kinematics." }, { "index": 10, "slug": "hugging-face-datasets", "entity_name": "Hugging Face Datasets & Hub", "entity_kind": "hub", "review_url": "https://bestaidata.org/reviews/hugging-face-datasets/", "official_url": "https://huggingface.co/datasets", "rank_label": "#1 Open ML Dataset Registry", "best_for": "Open pre-training streaming, academic benchmarks & Arrow/Parquet tooling", "neutrality_or_license": "Open Ecosystem Hub (per-repo licenses vary wildly)", "pricing": "Free public hosting/streaming; Pro/Enterprise Hub ($9–$50+/seat) for private repos", "turnaround_or_scale": "250,000+ public datasets · Petabyte Xet/Parquet CDN", "answer_in_brief": "Hugging Face Datasets Hub ranks #1 globally as the canonical open distribution registry for machine learning data, hosting 250,000+ repositories backed by Apache Arrow, automatic Parquet conversion, Croissant JSON-LD metadata, and chunk-deduplicated Xet storage. Its open `datasets` Python library and `hf://` fsspec filesystem allow zero-download streaming into PyTorch, DuckDB, and Polars. However, because anyone can upload unverified files with mislabeled license tags, enterprises must run strict SPDX, PII, and decontamination gates before ingestion." }, { "index": 11, "slug": "kaggle-datasets", "entity_name": "Kaggle Datasets & Competition Corpora", "entity_kind": "hub", "review_url": "https://bestaidata.org/reviews/kaggle-datasets/", "official_url": "https://www.kaggle.com/datasets", "rank_label": "#2 Community & Competition Hub", "best_for": "Tabular ML benchmarks, RSNA/medical challenge sets & Olympiad math evals", "neutrality_or_license": "Google subsidiary; competition rules often restrict commercial use", "pricing": "Free CLI/API download (`kagglehub`); competition rules govern commercial rights", "turnaround_or_scale": "350,000+ datasets (predominantly MB-to-low-GB CSV/ZIP archives)", "answer_in_brief": "Kaggle Datasets hosts over 350,000 community-uploaded datasets alongside world-class prize competition benchmarks such as ARC-Prize, AI Mathematical Olympiad (AIMO), and RSNA medical imaging challenges. Owned by Google, Kaggle excels at well-defined tabular, classical ML, and competition evaluation datasets backed by executable notebook baselines. However, for foundation LLM pre-training and enterprise production, Kaggle lags Hugging Face in petabyte Parquet streaming and suffers widespread unverified web-scraping among user-contributed CSVs." }, { "index": 12, "slug": "aws-data-exchange", "entity_name": "AWS Data Exchange (ADX)", "entity_kind": "hub", "review_url": "https://bestaidata.org/reviews/aws-data-exchange/", "official_url": "https://aws.amazon.com/data-exchange/", "rank_label": "#1 S3-Native Licensed Exchange", "best_for": "S3-hosted licensed news, financial, geospatial & healthcare grounding feeds", "neutrality_or_license": "Commercial bilateral Data Subscription Agreements (DSA)", "pricing": "Annual dataset subscriptions ($10K–$500K+/yr; AWS EDP eligible)", "turnaround_or_scale": "3,500+ catalogs · Direct S3 access points & Redshift shares", "answer_in_brief": "AWS Data Exchange (ADX) is Amazon's enterprise third-party data marketplace, hosting 3,500+ commercial and public data products from Reuters, Shutterstock, Change Healthcare, Equifax, and specialty providers with native delivery into Amazon S3, Redshift, and Lake Formation. Its core engineering advantage is zero-ETL S3 bucket mounting and consolidated AWS Enterprise Discount Program (EDP) billing. However, buyers must scrutinize each Data Subscription Agreement (DSA)—standard analytics licenses on ADX frequently prohibit training foundation model weights without an explicit AI Training Addendum." }, { "index": 13, "slug": "snowflake-marketplace", "entity_name": "Snowflake Marketplace", "entity_kind": "hub", "review_url": "https://bestaidata.org/reviews/snowflake-marketplace/", "official_url": "https://app.snowflake.com/marketplace", "rank_label": "#1 Structured & RAG Data Exchange", "best_for": "Enterprise RAG grounding, Text-to-SQL benchmarks & Data Clean Room joins", "neutrality_or_license": "Commercial marketplace terms + Cortex Knowledge Extension IP controls", "pricing": "Subscription + Snowflake compute credits ($0–$250K+/yr per feed)", "turnaround_or_scale": "2,700+ live shares · Instant zero-copy cross-cloud mounting", "answer_in_brief": "Snowflake Marketplace is the premier enterprise exchange for zero-copy structured, financial, retail, healthcare, and Cortex Knowledge Extension datasets, spanning 2,700+ live listings. Powered by Snowflake Secure Data Sharing and Data Clean Rooms, it lets teams evaluate and join third-party tables instantaneously without ETL pipelines or raw file transfers. It excels for enterprise RAG grounding, text-to-SQL agent evaluation, and regulated tabular ML, but is poorly suited to exporting petabytes of raw video, audio, or pre-training tokens to external GPU clusters." }, { "index": 14, "slug": "protege", "entity_name": "Protege (Licensed Multimodal & Clinical Data Exchange)", "entity_kind": "vendor", "review_url": "https://bestaidata.org/reviews/protege/", "official_url": "https://withprotege.ai", "rank_label": "#1 AI-Native Rights Licensing Platform", "best_for": "De-identified clinical DICOM/EHR, licensed video/motion & conversational audio", "neutrality_or_license": "100% Independent · Bilateral commercial AI training licenses", "pricing": "Custom per-modality volume license ($50K–$500K+ enterprise deals)", "turnaround_or_scale": "2–4 weeks from sample schema inspection to licensed S3 delivery", "answer_in_brief": "Protege ranks as the leading dedicated data-licensing intermediary purpose-built for AI model training, aggregating rights-cleared clinical (DICOM, pathology, EHR), conversational audio, and motion/video datasets directly from hospitals, media archives, and rights holders. Unlike general BI marketplaces where AI weight-training clauses are bolted on as an afterthought, every Protege catalog ships with explicit commercial AI training licenses and HIPAA Safe Harbor / Expert Determination de-identification. It excels at acquiring raw proprietary multimodal assets, which teams then enrich with expert annotation." } ], "head_to_head_comparisons": [ { "index": 1, "slug": "fineweb-vs-dolma-3", "title": "FineWeb vs. Dolma 3", "compare_url": "https://bestaidata.org/compare/fineweb-vs-dolma-3/", "best_for": "Selecting base LLM pre-training & mid-training annealing token mixes", "answer_in_brief": "Choose FineWeb (15T English + 8T FineWeb2 multilingual) when you need the largest pure web-HTML pre-training backbone across 1,000+ languages with pre-computed educational classifier scores. Choose Dolma 3 (5.2T+ tokens) when you want a complete, decontaminated multi-source pre-training mixture that already integrates linearized olmOCR scientific PDFs, code, and mathematical corpora alongside deterministic Olmo mixing scripts. In practice, many 2026 frontier baselines blend FineWeb2's multilingual shards with Dolma 3's scientific PDF and mid-training partitions." }, { "index": 2, "slug": "common-crawl-vs-fineweb", "title": "Common Crawl vs. FineWeb", "compare_url": "https://bestaidata.org/compare/common-crawl-vs-fineweb/", "best_for": "Deciding whether to run custom Spark/Ray HTML extraction on raw WARCs", "answer_in_brief": "For 95% of ML teams, FineWeb strictly dominates raw Common Crawl: it saves hundreds of thousands of CPU-hours in HTML parsing, adult/spam blocklisting, and MinHash deduplication while delivering measurably lower LLM validation perplexity than Common Crawl WET text. Re-ingesting raw Common Crawl WARC archives makes engineering sense in only three scenarios: extracting structured HTML/JSON-LD markup stripped by text parsers, capturing the freshest monthly crawls before FineWeb updates, or enforcing custom enterprise `robots.txt`/TDM opt-out allow-lists." }, { "index": 3, "slug": "hugging-face-vs-kaggle", "title": "Hugging Face Datasets vs. Kaggle Datasets", "compare_url": "https://bestaidata.org/compare/hugging-face-vs-kaggle/", "best_for": "Choosing where to source & host open training and benchmark datasets", "answer_in_brief": "Hugging Face Datasets Hub is the clear standard for LLM pre-training, post-training, vision-language, audio, and robotics pipelines thanks to native Arrow/Parquet streaming, chunk-deduplicated Xet storage, and multi-terabyte shard tooling. Kaggle Datasets wins specifically on sponsor-curated competition benchmarks—such as radiology DICOM challenges, tabular forecasting, ARC-Prize, and Olympiad math—where ground-truth labels were validated for cash-prize leaderboards. Both platforms require strict legal screening because community uploads frequently misstate upstream copyright licenses." }, { "index": 4, "slug": "aws-data-exchange-vs-snowflake-marketplace", "title": "AWS Data Exchange vs. Snowflake Marketplace", "compare_url": "https://bestaidata.org/compare/aws-data-exchange-vs-snowflake-marketplace/", "best_for": "Enterprise procurement of licensed third-party commercial datasets", "answer_in_brief": "Choose AWS Data Exchange (ADX) when acquiring unstructured or semi-structured licensed files (Parquet blobs, PDFs, imagery, audio, clinical archives) that need to stream directly from Amazon S3 into SageMaker or EKS GPU training clusters using committed AWS EDP spend. Choose Snowflake Marketplace when your primary AI workload is grounded enterprise RAG (via Cortex Knowledge Extensions), multi-table Text-to-SQL evaluation, or privacy-governed Data Clean Room joins over live relational tables. On both platforms, verify explicit foundation-model weight-training clauses before training." }, { "index": 5, "slug": "open-source-corpora-vs-custom-expert-data", "title": "Open-Source Corpora vs. Custom Expert Datasets", "compare_url": "https://bestaidata.org/compare/open-source-corpora-vs-custom-expert-data/", "best_for": "Allocating AI data budgets between free open shards & custom human SOWs", "answer_in_brief": "Open-source corpora (Dolma 3, FineWeb, The Stack v2, DROID) and custom human-authored datasets serve complementary, non-interchangeable stages of the modern AI stack. Open corpora provide 95%+ of raw pre-training volume at near-zero marginal token cost to establish general linguistic, world-knowledge, and syntax priors, but saturate rapidly due to benchmark contamination, CC-BY-NC/synthetic license traps, and missing tacit expert reasoning. Custom human datasets—commissioned under work-for-hire IP indemnity with >=0.85 IAA—drive virtually 100% of frontier differentiation in post-training, vertical domain mastery, and private holdout evaluation." }, { "index": 6, "slug": "agibot-world-vs-droid", "title": "AgiBot World vs. DROID", "compare_url": "https://bestaidata.org/compare/agibot-world-vs-droid/", "best_for": "Selecting open robot manipulation trajectories for VLA policy training", "answer_in_brief": "The choice between AgiBot World and DROID hinges immediately on commercial licensing and robot morphology: DROID (76,000 trajectories across 564 diverse real-world rooms) is licensed under permissive CC-BY-4.0, making it the mandatory open choice for commercial robotics companies building closed or proprietary VLA models. AgiBot World offers 13x higher trajectory volume (1M+ bimanual dual-arm episodes with tactile and failure-recovery traces), but its strict CC-BY-NC-SA 4.0 license restricts usage to non-commercial academic research unless you negotiate a separate bilateral license." } ], "foundation_dataset_benchmarks_2026": [ { "id": 1, "dataset": "Dolma 3 Mix", "curator": "Allen Institute for AI (Ai2)", "modality": "Pre-Training Text", "scale": "5.2T+ tokens (web, academic PDFs, code, math)", "top_level_license": "ODC-By 1.0 (upstream source terms apply)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "ODC-By wrapper covers curation, not underlying publisher copyright or robots.txt retro-revocations.", "review_url": "https://bestaidata.org/reviews/dolma-3/", "source_url": "https://huggingface.co/allenai" }, { "id": 2, "dataset": "FineWeb (v1.3)", "curator": "Hugging Face", "modality": "Pre-Training Text", "scale": "15T+ tokens (96+ Common Crawl snapshots)", "top_level_license": "ODC-By 1.0", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Web-scraped HTML retains third-party copyright claims in EU DSM Art. 4 opt-out audits.", "review_url": "https://bestaidata.org/reviews/fineweb/", "source_url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb" }, { "id": 3, "dataset": "FineWeb-Edu", "curator": "Hugging Face", "modality": "Pre-Training Text", "scale": "1.3T & 5.4T educational-filtered tokens", "top_level_license": "ODC-By 1.0 (Llama-3 classifier annotations)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Classifier bias over-weights textbook prose while starving conversational, tabular, and edge-case reasoning.", "review_url": "https://bestaidata.org/reviews/fineweb/", "source_url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu" }, { "id": 4, "dataset": "FineWeb2 (Multilingual)", "curator": "Hugging Face", "modality": "Pre-Training Text", "scale": "8T+ tokens across 1,000+ languages", "top_level_license": "ODC-By 1.0", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Low-resource language shards exhibit severe OCR/machine-translation boilerplate contamination.", "review_url": "https://bestaidata.org/reviews/fineweb/", "source_url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-2" }, { "id": 5, "dataset": "FinePDFs", "curator": "Hugging Face", "modality": "Pre-Training Text", "scale": "1.8T+ tokens parsed from scientific/technical PDFs", "top_level_license": "ODC-By 1.0 (document-level rights vary)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Multi-column table linearization errors and embedded publisher All-Rights-Reserved PDFs.", "review_url": null, "source_url": "https://huggingface.co/HuggingFaceFW" }, { "id": 6, "dataset": "Nemotron-CC (v2.1)", "curator": "NVIDIA", "modality": "Pre-Training Text", "scale": "6.3T tokens (4.4T real + 1.9T synthetic rephrased)", "top_level_license": "NVIDIA Open Model / Common Crawl Terms", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Synthetic rephrased splits carry upstream generator license notices and web provenance obligations.", "review_url": "https://bestaidata.org/reviews/nemotron-cc/", "source_url": "https://huggingface.co/nvidia" }, { "id": 7, "dataset": "Common Corpus", "curator": "PleIAs / Cultural Heritage Consortium", "modality": "Pre-Training Text", "scale": "2T+ tokens (public domain & permissive open texts)", "top_level_license": "Public Domain / CC-BY / CC0", "commercial_production_clearance": "Yes", "primary_enterprise_limitation": "Skews heavily historical/archival; lacks modern post-2024 API frameworks and contemporary syntax.", "review_url": null, "source_url": "https://huggingface.co/PleIAs" }, { "id": 8, "dataset": "Common Crawl Raw WARC/WET", "curator": "Common Crawl Foundation", "modality": "Pre-Training Text", "scale": "250B+ web pages (~400+ TiB per crawl)", "top_level_license": "CC Terms of Use (raw crawl access only)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Unfiltered PII, SEO spam, adult content, and zero copyright indemnity without 6-stage deduplication.", "review_url": "https://bestaidata.org/reviews/common-crawl/", "source_url": "https://commoncrawl.org" }, { "id": 9, "dataset": "RedPajama-Data-v2", "curator": "Together AI", "modality": "Pre-Training Text", "scale": "30T+ raw tokens w/ 40+ quality signals across 5 languages", "top_level_license": "Apache 2.0 scripts / Common Crawl upstream", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Delivers uncleaned indices + quality metadata; teams must run their own MinHash and PII scrubbing.", "review_url": "https://bestaidata.org/reviews/redpajama-v2/", "source_url": "https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2" }, { "id": 10, "dataset": "TxT360", "curator": "LLM360", "modality": "Pre-Training Text", "scale": "15T+ tokens (globally deduplicated across 99 CC dumps + ArXiv/FreeLaw)", "top_level_license": "ODC-By 1.0", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Cross-corpus global deduplication still leaves benchmark paraphrase overlap against MMLU/GPQA.", "review_url": null, "source_url": "https://huggingface.co/LLM360" }, { "id": 11, "dataset": "DCLM-Baseline (DataComp-LM)", "curator": "DataComp-LM Consortium", "modality": "Pre-Training Text", "scale": "4T filtered tokens from 240T pool", "top_level_license": "CC-BY-4.0 metadata / Common Crawl upstream", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "fastText OH-2.5 + ELI5 classifier biases output toward Q&A forum cadence.", "review_url": null, "source_url": "https://huggingface.co/mlfoundations" }, { "id": 12, "dataset": "CulturaX", "curator": "uonlp", "modality": "Pre-Training Text", "scale": "6.3T tokens across 167 languages (mC4 + OSCAR)", "top_level_license": "mC4 / OSCAR upstream terms", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "OSCAR registration gates and heterogeneous national copyright regimes across 167 locales.", "review_url": null, "source_url": "https://huggingface.co/datasets/uonlp/CulturaX" }, { "id": 13, "dataset": "Llama-Nemotron Post-Training", "curator": "NVIDIA", "modality": "Post-Training SFT / Pref", "scale": "30M+ SFT & reasoning traces (math, code, tool-use)", "top_level_license": "CC-BY-4.0 + Llama / DeepSeek / Qwen upstream clauses", "commercial_production_clearance": "Training-Only", "primary_enterprise_limitation": "Contains distilled synthetic outputs subject to upstream model-family naming and acceptable-use clauses.", "review_url": null, "source_url": "https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset" }, { "id": 14, "dataset": "Dolci-Instruct-SFT (Olmo 3)", "curator": "Allen Institute for AI (Ai2)", "modality": "Post-Training SFT / Pref", "scale": "1.8M+ curated multi-turn SFT & reasoning dialogues", "top_level_license": "ODC-By 1.0 (sub-mixture licenses vary)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Certain synthetic sub-splits inherit non-competing or attribution clauses from teacher models.", "review_url": null, "source_url": "https://huggingface.co/allenai" }, { "id": 15, "dataset": "Tulu 3 SFT Mixture", "curator": "Allen Institute for AI (Ai2)", "modality": "Post-Training SFT / Pref", "scale": "939K SFT prompts & completions across 18 sources", "top_level_license": "ODC-By 1.0 (composite upstream licenses)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Includes WildChat/ShareGPT-style & GPT-distilled subsets blocked by strict enterprise legal teams.", "review_url": null, "source_url": "https://huggingface.co/datasets/allenai/tulu-3-sft-mixture" }, { "id": 16, "dataset": "HelpSteer2 & HelpSteer3", "curator": "NVIDIA", "modality": "Post-Training SFT / Pref", "scale": "21K–40K+ human multi-attribute preference & edit pairs", "top_level_license": "CC-BY-4.0", "commercial_production_clearance": "Yes", "primary_enterprise_limitation": "Small total row volume; saturates quickly on repository-scale engineering and domain verticals.", "review_url": null, "source_url": "https://huggingface.co/datasets/nvidia/HelpSteer2" }, { "id": 17, "dataset": "UltraFeedback (Binarized)", "curator": "OpenBMB / Hugging Face H4", "modality": "Post-Training SFT / Pref", "scale": "64K prompts / 256K LLM-judged completions", "top_level_license": "MIT", "commercial_production_clearance": "Training-Only", "primary_enterprise_limitation": "GPT-4 judge annotations trigger OpenAI Terms-of-Service restrictions for competing commercial models.", "review_url": null, "source_url": "https://huggingface.co/datasets/HuggingFaceH4/ultrafeedback_binarized" }, { "id": 18, "dataset": "WildChat-1M", "curator": "Allen Institute for AI / Academic", "modality": "Post-Training SFT / Pref", "scale": "1M real user-chatbot multi-turn conversations", "top_level_license": "ODC-By 1.0 (OpenAI API upstream outputs)", "commercial_production_clearance": "Non-Commercial", "primary_enterprise_limitation": "Contains unredacted residual PII risk and OpenAI-generated assistant turns.", "review_url": null, "source_url": "https://huggingface.co/datasets/allenai/WildChat-1M" }, { "id": 19, "dataset": "LMSYS-Chat-1M", "curator": "LMSYS Org", "modality": "Post-Training SFT / Pref", "scale": "1M arena conversations across 25 LLMs", "top_level_license": "LMSYS Non-Commercial Research License", "commercial_production_clearance": "Non-Commercial", "primary_enterprise_limitation": "Explicitly restricted to non-commercial academic research; prohibited in commercial model weights.", "review_url": null, "source_url": "https://huggingface.co/datasets/lmsys/lmsys-chat-1m" }, { "id": 20, "dataset": "OpenCodeReasoning", "curator": "NVIDIA", "modality": "Code & SWE", "scale": "736K+ competitive programming chain-of-thought solutions", "top_level_license": "CC-BY-4.0", "commercial_production_clearance": "Training-Only", "primary_enterprise_limitation": "Single-file algorithmic contests only; zero multi-file git repo architecture or CI integration.", "review_url": null, "source_url": "https://huggingface.co/datasets/nvidia/OpenCodeReasoning" }, { "id": 21, "dataset": "The Stack v2", "curator": "BigCode / Software Heritage", "modality": "Code & SWE", "scale": "67.5B files (~900B+ tokens across 619 languages)", "top_level_license": "Per-file SPDX (permissive + opt-out governance)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Requires downloading raw blobs from Software Heritage S3 and honoring ongoing author opt-out removals.", "review_url": null, "source_url": "https://huggingface.co/datasets/bigcode/the-stack-v2" }, { "id": 22, "dataset": "StarCoder2Data / Stack-Edu", "curator": "BigCode / Hugging Face", "modality": "Code & SWE", "scale": "125B–900B educational & permissive code tokens", "top_level_license": "Gated BigCode OpenRAIL-M / Per-File SPDX", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Copyleft GPL/AGPL file leakage risk if strict SPDX license allow-lists are not enforced pre-training.", "review_url": null, "source_url": "https://huggingface.co/bigcode" }, { "id": 23, "dataset": "CommitPackFT", "curator": "BigCode", "modality": "Code & SWE", "scale": "702K filtered git commit-message-to-diff pairs (277 langs)", "top_level_license": "MIT / Apache-2.0 / BSD permissive subset", "commercial_production_clearance": "Yes", "primary_enterprise_limitation": "Short single-commit diffs lacking multi-commit architectural refactors or executable test harnesses.", "review_url": null, "source_url": "https://huggingface.co/datasets/bigcode/commitpackft" }, { "id": 24, "dataset": "SWE-bench Verified (Eval Split)", "curator": "OpenAI / Princeton NLP", "modality": "Code & SWE", "scale": "500 human-validated GitHub issue resolution tasks", "top_level_license": "MIT", "commercial_production_clearance": "Yes", "primary_enterprise_limitation": "Evaluation holdout ONLY (500 rows); training on it invalidates public coding benchmark integrity.", "review_url": null, "source_url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified" }, { "id": 25, "dataset": "LAION-BVD (Big Video Dataset)", "curator": "LAION Ecosystem", "modality": "Vision & Multimodal", "scale": "100M+ video-text clip URLs & spatio-temporal captions", "top_level_license": "CC-BY-4.0 metadata index (videos unhosted)", "commercial_production_clearance": "Non-Commercial", "primary_enterprise_limitation": "URL-only index pointing to copyrighted web video streams; link rot (>25%) and zero commercial video rights.", "review_url": "https://bestaidata.org/reviews/laion-bvd/", "source_url": "https://laion.ai" }, { "id": 26, "dataset": "PixMo", "curator": "Allen Institute for AI (Ai2)", "modality": "Vision & Multimodal", "scale": "2.3M+ dense voice-transcribed image captions, points & QA", "top_level_license": "ODC-By 1.0", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Annotations are open, but underlying web image URLs still require commercial image-rights clearance.", "review_url": null, "source_url": "https://huggingface.co/allenai" }, { "id": 27, "dataset": "SA-1B (Segment Anything)", "curator": "Meta AI", "modality": "Vision & Multimodal", "scale": "11M licensed images / 1.1B segmentation masks", "top_level_license": "SA-1B Research License", "commercial_production_clearance": "Non-Commercial", "primary_enterprise_limitation": "Research-only license strictly forbids training commercial production foundation models.", "review_url": null, "source_url": "https://ai.meta.com/datasets/segment-anything/" }, { "id": 28, "dataset": "Objaverse-XL", "curator": "Allen Institute for AI (Ai2)", "modality": "Vision & Multimodal", "scale": "10.2M 3D objects & meshes (GitHub, Thingiverse, Sketchfab)", "top_level_license": "ODC-By 1.0 index (per-asset CC-BY / CC-BY-NC mix)", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Over 40% of crawled 3D meshes carry Non-Commercial or No-Derivatives creator licenses.", "review_url": null, "source_url": "https://objaverse.allenai.org" }, { "id": 29, "dataset": "NVIDIA Granary (v2)", "curator": "NVIDIA NeMo", "modality": "Speech & Audio", "scale": "1M+ hours multilingual ASR & AST speech (25 European langs)", "top_level_license": "CC-BY-4.0 pipeline / YouTube & VoxPopuli upstream", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Pseudo-labeled speech transcripts lack studio-grade speaker consent and paralinguistic emotion labels.", "review_url": "https://bestaidata.org/reviews/nvidia-granary/", "source_url": "https://huggingface.co/nvidia" }, { "id": 30, "dataset": "People's Speech", "curator": "MLCommons", "modality": "Speech & Audio", "scale": "30,000+ hours English speech w/ CC-BY / CC-BY-SA provenance", "top_level_license": "CC-BY-4.0 / CC-BY-SA", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "CC-BY-SA sub-shards trigger ShareAlike legal review; predominantly monologue civic/lecture audio.", "review_url": null, "source_url": "https://mlcommons.org/datasets/peoples-speech/" }, { "id": 31, "dataset": "Mozilla Common Voice 27.0", "curator": "Mozilla Foundation", "modality": "Speech & Audio", "scale": "33,000+ validated hours across 130+ languages", "top_level_license": "CC0 1.0 Public Domain Dedication", "commercial_production_clearance": "Yes", "primary_enterprise_limitation": "Short isolated sentence read-prompts; lacks overlapping conversational turn-taking and noisy telephony.", "review_url": null, "source_url": "https://commonvoice.mozilla.org" }, { "id": 32, "dataset": "Emilia-Dataset", "curator": "Amphion / Open Audio", "modality": "Speech & Audio", "scale": "101,000+ hours in-the-wild speech (6 languages)", "top_level_license": "CC-BY-NC-4.0", "commercial_production_clearance": "Non-Commercial", "primary_enterprise_limitation": "Strict CC-BY-NC-4.0 license bars commercial TTS/voice-LLM deployment without separate rights acquisition.", "review_url": null, "source_url": "https://huggingface.co/datasets/amphion/Emilia-Dataset" }, { "id": 33, "dataset": "AgiBot World (Beta / Colosseo)", "curator": "AgiBot Research", "modality": "Robotics & Embodied", "scale": "1M+ bimanual manipulation trajectories (217 tasks, 100+ robots)", "top_level_license": "CC-BY-NC-SA 4.0 (Open Research Release)", "commercial_production_clearance": "Non-Commercial", "primary_enterprise_limitation": "CC-BY-NC-SA blocks commercial humanoid/industrial deployment without bilateral commercial licensing.", "review_url": "https://bestaidata.org/reviews/agibot-world/", "source_url": "https://huggingface.co/agibot-world" }, { "id": 34, "dataset": "DROID (Distributed Robot Interaction)", "curator": "TRI / Stanford / Berkeley / Consortium", "modality": "Robotics & Embodied", "scale": "76,000 teleoperated trajectories (350 hours, 564 scenes)", "top_level_license": "CC-BY-4.0", "commercial_production_clearance": "Yes", "primary_enterprise_limitation": "Locked to single-arm Franka Panda + Robotiq gripper kinematics; fails to transfer directly to dexterous hands.", "review_url": "https://bestaidata.org/reviews/droid-robotics/", "source_url": "https://droid-dataset.github.io" }, { "id": 35, "dataset": "Open X-Embodiment (OXE)", "curator": "Google DeepMind & 34 Academic Labs", "modality": "Robotics & Embodied", "scale": "1M+ episodes across 22 robot embodiments (RLDS format)", "top_level_license": "Composite Apache 2.0 / CC-BY / Custom Lab Licenses", "commercial_production_clearance": "Mixed", "primary_enterprise_limitation": "Heterogeneous camera intrinsics, control frequencies (3Hz–50Hz), and mixed academic sub-dataset licenses.", "review_url": null, "source_url": "https://robotics-transformer-x.github.io" }, { "id": 36, "dataset": "ToyBlocks (Embodied Spatial & Assembly)", "curator": "Embodied Vision Benchmark Consortium", "modality": "Robotics & Embodied", "scale": "120K+ fine-grained 6-DoF assembly & contact-rich episodes", "top_level_license": "CC-BY-NC-4.0", "commercial_production_clearance": "Non-Commercial", "primary_enterprise_limitation": "Tabletop rigid-block domain under-represents deformable objects, kitchen fluids, and industrial tolerances.", "review_url": null, "source_url": "https://huggingface.co/datasets" } ] }