diff --git a/docs/docs/api/index.md b/docs/docs/api/index.md index 2a3c298efd..55acaa4590 100644 --- a/docs/docs/api/index.md +++ b/docs/docs/api/index.md @@ -16,7 +16,6 @@ This reference documents the Python API of Deep Lake. - [Query Language](query.md): Complete TQL syntax and operations - [Metadata](metadata.md): Dataset and column metadata management - [Version Control](version_control.md): Version control, history, branches, and tags -- [Miscellaneous](misc.md): Additional auxiliary functionality ## Index Management diff --git a/docs/docs/api/query.md b/docs/docs/api/query.md index 0f8cd46af4..162df87df1 100644 --- a/docs/docs/api/query.md +++ b/docs/docs/api/query.md @@ -229,7 +229,7 @@ results = deeplake.query(""" l.label FROM "s3://bucket1/images" AS i JOIN "gcs://bucket2/embeddings" AS e ON i.id = e.image_id - JOIN "azure://container/labels" AS l ON i.id = l.image_id + JOIN "azure:////labels" AS l ON i.id = l.image_id WHERE l.confidence > 0.9 ORDER BY COSINE_SIMILARITY(e.embedding, ARRAY[0.1, 0.2, 0.3]) DESC LIMIT 100 diff --git a/docs/docs/concepts/.pages b/docs/docs/concepts/.pages deleted file mode 100644 index c764a8509a..0000000000 --- a/docs/docs/concepts/.pages +++ /dev/null @@ -1,4 +0,0 @@ -nav: - - Architecture: architecture.md - - Data Model: data-model.md - - Performance: performance.md \ No newline at end of file diff --git a/docs/docs/getting-started/quickstart.md b/docs/docs/getting-started/quickstart.md index c50f79a377..940b2f4506 100644 --- a/docs/docs/getting-started/quickstart.md +++ b/docs/docs/getting-started/quickstart.md @@ -55,7 +55,7 @@ ds = deeplake.create("path/to/dataset") # Or create in cloud storage ds = deeplake.create("s3://my-bucket/dataset") ds = deeplake.create("gcs://my-bucket/dataset") -ds = deeplake.create("azure://container/dataset") +ds = deeplake.create("azure:////") ``` ## Adding Data @@ -179,8 +179,8 @@ cats = future.result() ## Next Steps -- Explore [RAG applications](../../guide/rag) -- Check out [Deep Learning integration](../../guide/deep-learning/deep-learning) +- Explore [RAG applications](../guide/rag.md) +- Check out [Deep Learning integration](../guide/deep-learning/deep-learning.ipynb) ## Support diff --git a/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/cors.md b/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/cors.md index 0d8f1640c9..2d1459f678 100644 --- a/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/cors.md +++ b/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/cors.md @@ -18,7 +18,7 @@ In order to visualize Deep Lake datasets stored in your own S3 buckets in the [D "HEAD" ], "AllowedOrigins": [ - "*.activeloop.ai" + "https://*.activeloop.ai" ], "ExposeHeaders": [] } diff --git a/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/provisioning.md b/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/provisioning.md index b0fd251e08..33f5e84c27 100644 --- a/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/provisioning.md +++ b/docs/docs/getting-started/storage-and-creds/managed-credentials/aws/provisioning.md @@ -129,7 +129,7 @@ The most secure method for connecting data from your AWS account to Deep Lake is "kms:DescribeKey" ], "Resource": [ - "YOUR_KMS_KEY_ARN” + "YOUR_KMS_KEY_ARN" ] } ] diff --git a/docs/docs/getting-started/storage-and-creds/managed-credentials/azure/cors.md b/docs/docs/getting-started/storage-and-creds/managed-credentials/azure/cors.md index cc67bcf7e5..f474273248 100644 --- a/docs/docs/getting-started/storage-and-creds/managed-credentials/azure/cors.md +++ b/docs/docs/getting-started/storage-and-creds/managed-credentials/azure/cors.md @@ -15,11 +15,11 @@ Cross-Origin Resource Sharing (CORS) is typically enabled by default in Azure. I 3\. Open the `Resource sharing (CORS)` section on the left nav. -
+
4\. Add the following items to the permissions. -
+
| Allowed origins | Allowed methods | Allowed headers | | ------------------------- | --------------- | --------------- | diff --git a/docs/docs/getting-started/storage-and-creds/managed-credentials/gcp/provisioning.md b/docs/docs/getting-started/storage-and-creds/managed-credentials/gcp/provisioning.md index ca96b7ed8f..2e480ed5a2 100644 --- a/docs/docs/getting-started/storage-and-creds/managed-credentials/gcp/provisioning.md +++ b/docs/docs/getting-started/storage-and-creds/managed-credentials/gcp/provisioning.md @@ -7,7 +7,7 @@ description: "Provision Google Cloud Resources For Use With Deep Lake, Enabling ## Setting up Federated Credentials in Google Cloud Platform -The most secure method for connecting data from your Azure storage to Deep Lake is using Federated Credentials, which are set up using the steps below: +The most secure method for connecting data from your Google Cloud Storage bucket to Deep Lake is using Federated Credentials, which are set up using the steps below: #### Step 1: Create Google Cloud Service Account diff --git a/docs/docs/guide/deep-learning/async-data-loader.md b/docs/docs/guide/deep-learning/async-data-loader.md index 0d109c79b1..2866ed041b 100644 --- a/docs/docs/guide/deep-learning/async-data-loader.md +++ b/docs/docs/guide/deep-learning/async-data-loader.md @@ -66,11 +66,13 @@ lock = Lock() index = -1 def generate_data(ds: deeplake.Dataset): total_count = len(ds) + if total_count == 0: + return global index while True: idx = 0 with lock: - index = (index + 1) % (total_count - 1) + index = (index + 1) % total_count idx = index yield ds[idx] diff --git a/docs/docs/guide/deep-learning/deep-learning.ipynb b/docs/docs/guide/deep-learning/deep-learning.ipynb index 332ccca3c3..f2b8a0c7ef 100644 --- a/docs/docs/guide/deep-learning/deep-learning.ipynb +++ b/docs/docs/guide/deep-learning/deep-learning.ipynb @@ -175,8 +175,8 @@ "source": [ "# Download dataset\n", "from IPython.display import clear_output\n", - "# !wget https://github.com/activeloopai/examples/blob/main/colabs/starting_data/animals.tar\n", - "!curl -L -o animals.tar https://github.com/activeloopai/examples/blob/main/colabs/starting_data/animals.tar" + "# !wget https://raw.githubusercontent.com/activeloopai/examples/main/colabs/starting_data/animals.tar\n", + "!curl -L -o animals.tar https://raw.githubusercontent.com/activeloopai/examples/main/colabs/starting_data/animals.tar" ] }, { @@ -246,7 +246,7 @@ "outputs": [], "source": [ "# Find the class_names and list of files that need to be uploaded\n", - "dataset_folder = '/Users/istranic/ActiveloopCode/Datasets/animals'\n", + "dataset_folder = 'animals'\n", "\n", "# Find the subfolders, but filter additional files like DS_Store that are added on Mac machines.\n", "class_names = [item for item in os.listdir(dataset_folder) if os.path.isdir(os.path.join(dataset_folder, item))]\n", diff --git a/docs/docs/guide/deep-learning/mmseg.md b/docs/docs/guide/deep-learning/mmseg.md index a8452ba963..74c3257dc5 100644 --- a/docs/docs/guide/deep-learning/mmseg.md +++ b/docs/docs/guide/deep-learning/mmseg.md @@ -111,7 +111,7 @@ data = dict( ), val=dict( pipeline=test_pipeline, - deeplake_path="hub://activeloop/coco-val-seg-mask"", + deeplake_path="hub://activeloop/coco-val-seg-mask", deeplake_tensors = {"img": "images", "gt_semantic_seg": "seg_masks"}, deeplake_dataloader={"shuffle": False, "batch_size": 1, "num_workers": 0, "drop_last": True} ) diff --git a/docs/docs/guide/rag.md b/docs/docs/guide/rag.md index 7106ef92d6..c04add25cd 100644 --- a/docs/docs/guide/rag.md +++ b/docs/docs/guide/rag.md @@ -580,7 +580,7 @@ class Document(BaseModel): data: Dict[str, Any] score: Optional[float] = None -def softmax(retrieved_score: list[float], max_weight: int = 700) -> Dict[str, Document]: +def softmax(retrieved_score: list[float], max_weight: int = 700) -> list[float]: # Compute the exponentials exp_scores = [math.exp(min(score, max_weight)) for score in retrieved_score] @@ -672,7 +672,7 @@ def fusion(docs_vs: Dict[str, Document], docs_bm25: Dict[str, Document]) -> Dict VECTOR_WEIGHT = 0.5 LEXICAL_WEIGHT = 0.5 - results: Dict[str, Dict[str, Document]] = {} + results: Dict[str, Document] = {} for k in set(docs_vs) | set(docs_bm25): @@ -691,16 +691,16 @@ print(results) Output: ``` -{'2637': Document(id='2637', data={'restaurant_name': 'Mifen101 花溪米粉王', 'restaurant_review': 'Feel like I’m back in China.'}, score=0.013747293509625419), - '5136': Document(id='5136', data={'restaurant_name': 'Scratch', 'restaurant_review': 'Just had drinks. They were good!'}, score=0.024505473374994282), - '17426': Document(id='17426', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks an easy going bartenders'}, score=0.024579178342433523), - '17444': Document(id='17444', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks, good food'}, score=0.02475096426920331), - '2496': Document(id='2496', data={'restaurant_name': 'Seasons Noodles & Dumplings Garden', 'restaurant_review': 'Comfort food, excellent service! Feel like back to home.'}, score=0.005430922978171003), - '4022': Document(id='4022', data={'restaurant_name': 'Eureka! Mountain View', 'restaurant_review': 'Good drinks and burgers'}, score=0.0246334319067476), - '3518': Document(id='3518', data={'restaurant_name': 'Olympus Caffe & Bakery', 'restaurant_review': 'I like the garden to sit down with friends and have a drink.'}, score=0.08915287739833623), - '17502': Document(id='17502', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Nice place for a drink'}, score=0.02653095210662131), - '11383': Document(id='11383', data={'restaurant_name': 'Ludwigs Biergarten Mountain View', 'restaurant_review': 'Beer is fresh tables are big feel like a proper beer garden'}, score=0.011447537567869991), - '10788': Document(id='10788', data={'restaurant_name': 'Casa Lupe', 'restaurant_review': 'Run by a family that makes you feel like part of the family. Awesome food. I love their wet Chili Verde burritos'}, score=0.00522136854599736)} +{'2637': Document(id='2637', data={'restaurant_name': 'Mifen101 花溪米粉王', 'restaurant_review': 'Feel like I’m back in China.'}, score=0.054989174038501676), + '5136': Document(id='5136', data={'restaurant_name': 'Scratch', 'restaurant_review': 'Just had drinks. They were good!'}, score=0.09802189349997713), + '17426': Document(id='17426', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks an easy going bartenders'}, score=0.09831671336973409), + '17444': Document(id='17444', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Good drinks, good food'}, score=0.09900385707681324), + '2496': Document(id='2496', data={'restaurant_name': 'Seasons Noodles & Dumplings Garden', 'restaurant_review': 'Comfort food, excellent service! Feel like back to home.'}, score=0.02172369191268401), + '4022': Document(id='4022', data={'restaurant_name': 'Eureka! Mountain View', 'restaurant_review': 'Good drinks and burgers'}, score=0.0985337276269904), + '3518': Document(id='3518', data={'restaurant_name': 'Olympus Caffe & Bakery', 'restaurant_review': 'I like the garden to sit down with friends and have a drink.'}, score=0.3566115095933449), + '17502': Document(id='17502', data={'restaurant_name': "St. Stephen's Green", 'restaurant_review': 'Nice place for a drink'}, score=0.10612380842648524), + '11383': Document(id='11383', data={'restaurant_name': 'Ludwigs Biergarten Mountain View', 'restaurant_review': 'Beer is fresh tables are big feel like a proper beer garden'}, score=0.045790150271479965), + '10788': Document(id='10788', data={'restaurant_name': 'Casa Lupe', 'restaurant_review': 'Run by a family that makes you feel like part of the family. Awesome food. I love their wet Chili Verde burritos'}, score=0.02088547418398944)} ``` We sort the results dictionary by each document's combined score in descending order, ensuring that the highest-ranking documents appear first. @@ -856,6 +856,7 @@ for el, score in zip(bm25_results, bm25s): ```python results = fusion(docs_vs, docs_bm25) +sorted_documents = dict(sorted(results.items(), key=lambda item: item[1].score, reverse=True)) ``` ```python @@ -1287,8 +1288,7 @@ Dataset(columns=(text,embedding), length=19719) ``` ```python -medical_dataset.add_column(name="embedding", dtype=types.Array(types.Float32(),dimensions=2)) -medical_dataset.commit() +# The copied dataset already has an embedding column, so reuse it. ``` 3. **Text Extraction** : diff --git a/docs/docs/guide/v3-conversion.md b/docs/docs/guide/v3-conversion.md index eb92c10aa0..7f5380afe4 100644 --- a/docs/docs/guide/v3-conversion.md +++ b/docs/docs/guide/v3-conversion.md @@ -105,6 +105,8 @@ deeplake.create = create ```python # 1. Create v4 dataset with desired schema +old_ds_path = "hub://org_name/v3_dataset" # Replace with the v3 dataset path to migrate. +batch_size = 32 ds = deeplake.create("s3://new/dataset") ds.add_column("embeddings", deeplake.types.Embedding(768)) ds.add_column("images", deeplake.types.Image()) @@ -115,9 +117,10 @@ source = deeplake.query(f'SELECT * FROM "{old_ds_path}"') # 3. Migrate in batches with progress tracking for i in range(0, len(source), batch_size): - batch = source[i:i+batch_size] + batch = [row.to_dict() for row in source[i:i+batch_size]] ds.append(batch) - if i % 10000 == 0: + processed = i + len(batch) + if processed // 10000 > i // 10000 or processed == len(source): ds.commit() ``` diff --git a/docs/docs/llms.txt b/docs/docs/llms.txt index 885ff5e876..ad5a57fa22 100644 --- a/docs/docs/llms.txt +++ b/docs/docs/llms.txt @@ -122,7 +122,7 @@ SELECT AVG(features, axis=0) FROM dataset SELECT i.image, e.embedding, m.metadata FROM "s3://bucket1/images" AS i JOIN "gcs://bucket2/embeddings" AS e ON i.id = e.image_id -JOIN "azure://container/meta" AS m ON i.id = m.image_id +JOIN "azure:////meta" AS m ON i.id = m.image_id WHERE m.verified = true ORDER BY COSINE_SIMILARITY(e.embedding, ARRAY[0.1,0.2,0.3]) DESC ```