Download user datasets

DownloadUserDatasets.py

import os
import tempfile
from math import ceil
from zipfile import ZipFile

from LOGS import LOGS
from LOGS.Entities import DatasetRequestParameter, PersonRequestParameter


def downloadUserDatasets(logs: LOGS, userName, targetDirectory):
    if not os.path.exists(targetDirectory):
        raise Exception("Temp path %a does not exist" % targetDirectory)

    if not os.path.isdir(targetDirectory):
        raise Exception("Temp path %a is not a directory" % targetDirectory)

    temporaryDirectory = tempfile.TemporaryDirectory()
    tempdir = temporaryDirectory.name

    user = logs.persons(PersonRequestParameter(login=userName)).first()

    if not user:
        raise Exception("Could not find a user with user name %a" % userName)

    count = logs.datasets(DatasetRequestParameter(operatorIds=[user.id])).count
    splitCount = 50
    partCount = ceil(count / splitCount)

    print("Downloading %d dataset(s) to director %a." % (count, targetDirectory))

    part = 0
    for datasets in logs.datasets(DatasetRequestParameter(operatorIds=[user.id])).split(
        splitCount
    ):
        part += 1
        zipName = "datasets_part_%05d" % part
        zipFileName = zipName + ".zip"
        # zipDir = os.path.join(tempdir, zipName)
        zipFile = os.path.join(tempdir, zipFileName)
        print("Downloading part %d of %d..." % (part, partCount))
        if not os.path.exists(zipFile):
            datasets.download(tempdir, zipFileName)
        print("  Unpacking part %d..." % part)
        with ZipFile(zipFile, "r") as zObject:
            zObject.extractall(path=targetDirectory)

    print(
        "Download of %d dataset(s) to director %a complete." % (count, targetDirectory)
    )
    temporaryDirectory.cleanup()


if __name__ == "__main__":
    logs = LOGS()

    targetDir = "target"
    if not os.path.exists(targetDir):
        os.mkdir(targetDir)

    if not os.path.isdir(targetDir):
        raise Exception("Temp path %a exists but is not a directory" % targetDir)

    downloadUserDatasets(logs, "test", targetDir)