Download vast amount of dataset files

DownloadVastAmountOfDatasetFiles.py

import tempfile
from datetime import datetime, timedelta

from LOGS import LOGS
from LOGS.Entities import DatasetRequestParameter

# Connect to LOGS
logs = LOGS()

# Create an temporary directory
targetDirectory = tempfile.mkdtemp()


# The attempt to download all datasets for e.g. the last year might result in an
# error as the zip file will be too large
# A solution is to split the iterator into multiple iterators using split()
part = 0
# split will generate an iterator of iterators each containing 50 of the large iterators elements
for datasets in logs.datasets(
    DatasetRequestParameter(dateAddedFrom=datetime.now() - timedelta(days=365))
).split(50):
    part += 1
    zipName = "datasets_part_%05d" % part
    print("Downloading %a..." % zipName)
    # We are using the iterator download functionality to create a zip file of the datsets bundles
    datasets.download(fileName=zipName)