Skip to content

google_download_util

download_file_from_google_drive

Download the data from the Google drive public URL.

This method will try to download the file for given number of retries till successful.

Parameters:

Name Type Description Default
file_id str

File ID of Google drive URL.

 required
destination str

Destination path where the data needs to be stored.

 required
max_retries int

max number of retries.

 3
Source code in fastestimator/fastestimator/util/google_download_util.py 
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
def download_file_from_google_drive(file_id: str, destination: str, max_retries: int = 3) -> None:
    """Download the data from the Google drive public URL.

    This method will try to download the file for given number of retries till successful.

    Args:
        file_id: File ID of Google drive URL.
        destination: Destination path where the data needs to be stored.
        max_retries: max number of retries.
    """
    if is_valid_file(destination):
        print(f"File {destination} already exists, skipping download.")
        return

    for _ in range(max_retries):
        if is_valid_file(destination):
            return
        # Randomize a sleep interval before attempting to download in order to prevent multiple unit tests from hitting
        # the drive simultaneously
        time.sleep(random.randint(5, 10))
        if is_valid_file(destination):
            # Check again in case some other thread came through and downloaded while you were sleeping
            return
        try:
            _download_file_from_google_drive(file_id=file_id, destination=destination)
        except Exception as e:
            print(f"Exception occurred while downloading {destination}, will try again", e)
    raise ValueError(f"Couldn't download {destination} after {max_retries} retries.")