mlvbench.cache¶

from pathlib import Path

from mlvbench.cache import (
    get_cached_resource,
    create_cached_resource,
    finalize_cached_resource,
)

def my_data(seed: int = 2) -> Path:
    # A cached resource is identified by a name, version and the parameters that where
    # used to generate the resource:
    name = "my_data"
    version = "1"
    parameters = { "seed": seed }

    # If the resource with the given parameters already exists, get_cached_resource will
    # return its path.
    path = get_cached_resource(name, version, parameters)
    if path is not None:
        return path

    # If the resource doesn't exist yet, create_cached_resource will create a temporary
    # directory where we can prepare the data.
    path = create_cached_resource(name, version, parameters)

    # Prepare the data at the given path
    ...

    # Once all files have been generated, finalize_cached_resource will move the data to
    # its final location. This method takes care of race conditions: If a different
    # process has generated the same resource in the meantime, the newly created data
    # will be discarded and the cached data is used.
    path = finalize_cached_resource(path)

    return path