Google Dataflow API . v1b3 . projects . jobs

Instance Methods

messages()

Returns the messages Resource.

workItems()

Returns the workItems Resource.

create(projectId, body, view=None)

Creates a dataflow job.

get(projectId, jobId, view=None)

Gets the state of the specified dataflow job.

getMetrics(projectId, jobId, startTime=None)

Request the job status.

list(projectId, pageToken=None, pageSize=None, view=None)

List the jobs of a project

list_next(previous_request, previous_response)

Retrieves the next page of results.

patch(projectId, jobId, body)

Updates the state of an existing dataflow job. This method supports patch semantics.

update(projectId, jobId, body)

Updates the state of an existing dataflow job.

Method Details

create(projectId, body, view=None)
Creates a dataflow job.

Args:
  projectId: string, A parameter (required)
  body: object, The request body. (required)
    The object takes the form of:

{
  "requestedState": "A String",
  "name": "A String",
  "projectId": "A String",
  "createTime": "A String",
  "environment": {
    "version": {
      "a_key": "",
    },
    "workerPools": [
      {
        "diskSourceImage": "A String",
        "teardownPolicy": "A String",
        "kind": "A String",
        "packages": [
          {
            "location": "A String",
            "name": "A String",
          },
        ],
        "machineType": "A String",
        "zone": "A String",
        "onHostMaintenance": "A String",
        "taskrunnerSettings": {
          "workflowFileName": "A String",
          "logUploadLocation": "A String",
          "commandlinesFileName": "A String",
          "alsologtostderr": True or False,
          "vmId": "A String",
          "baseTaskDir": "A String",
          "continueOnException": True or False,
          "baseUrl": "A String",
          "parallelWorkerSettings": {
            "reportingEnabled": True or False,
            "shuffleServicePath": "A String",
            "workerId": "A String",
            "baseUrl": "A String",
            "servicePath": "A String",
            "tempStoragePrefix": "A String",
          },
          "oauthScopes": [
            "A String",
          ],
          "taskGroup": "A String",
          "languageHint": "A String",
          "logToSerialconsole": True or False,
          "streamingWorkerMainClass": "A String",
          "logDir": "A String",
          "dataflowApiVersion": "A String",
          "taskUser": "A String",
          "harnessCommand": "A String",
          "tempStoragePrefix": "A String",
        },
        "diskSizeGb": 42,
        "numWorkers": 42,
        "defaultPackageSet": "A String",
        "metadata": {
          "a_key": "A String",
        },
        "autoscalingSettings": {
          "maxNumWorkers": 42,
          "algorithm": "A String",
        },
        "dataDisks": [
          {
            "mountPoint": "A String",
            "sizeGb": 42,
            "diskType": "A String",
          },
        ],
      },
    ],
    "dataset": "A String",
    "experiments": [
      "A String",
    ],
    "sdkPipelineOptions": {
      "a_key": "",
    },
    "userAgent": {
      "a_key": "",
    },
    "clusterManagerApiService": "A String",
    "tempStoragePrefix": "A String",
  },
  "steps": [
    {
      "kind": "A String",
      "name": "A String",
      "properties": {
        "a_key": "",
      },
    },
  ],
  "currentStateTime": "A String",
  "type": "A String",
  "id": "A String",
  "currentState": "A String",
  "executionInfo": {
    "stages": {
      "a_key": {
        "stepName": [
          "A String",
        ],
      },
    },
  },
}

  view: string, A parameter
    Allowed values
      JOB_VIEW_ALL - 
      JOB_VIEW_SUMMARY - 
      JOB_VIEW_UNKNOWN - 

Returns:
  An object of the form:

    {
    "requestedState": "A String",
    "name": "A String",
    "projectId": "A String",
    "createTime": "A String",
    "environment": {
      "version": {
        "a_key": "",
      },
      "workerPools": [
        {
          "diskSourceImage": "A String",
          "teardownPolicy": "A String",
          "kind": "A String",
          "packages": [
            {
              "location": "A String",
              "name": "A String",
            },
          ],
          "machineType": "A String",
          "zone": "A String",
          "onHostMaintenance": "A String",
          "taskrunnerSettings": {
            "workflowFileName": "A String",
            "logUploadLocation": "A String",
            "commandlinesFileName": "A String",
            "alsologtostderr": True or False,
            "vmId": "A String",
            "baseTaskDir": "A String",
            "continueOnException": True or False,
            "baseUrl": "A String",
            "parallelWorkerSettings": {
              "reportingEnabled": True or False,
              "shuffleServicePath": "A String",
              "workerId": "A String",
              "baseUrl": "A String",
              "servicePath": "A String",
              "tempStoragePrefix": "A String",
            },
            "oauthScopes": [
              "A String",
            ],
            "taskGroup": "A String",
            "languageHint": "A String",
            "logToSerialconsole": True or False,
            "streamingWorkerMainClass": "A String",
            "logDir": "A String",
            "dataflowApiVersion": "A String",
            "taskUser": "A String",
            "harnessCommand": "A String",
            "tempStoragePrefix": "A String",
          },
          "diskSizeGb": 42,
          "numWorkers": 42,
          "defaultPackageSet": "A String",
          "metadata": {
            "a_key": "A String",
          },
          "autoscalingSettings": {
            "maxNumWorkers": 42,
            "algorithm": "A String",
          },
          "dataDisks": [
            {
              "mountPoint": "A String",
              "sizeGb": 42,
              "diskType": "A String",
            },
          ],
        },
      ],
      "dataset": "A String",
      "experiments": [
        "A String",
      ],
      "sdkPipelineOptions": {
        "a_key": "",
      },
      "userAgent": {
        "a_key": "",
      },
      "clusterManagerApiService": "A String",
      "tempStoragePrefix": "A String",
    },
    "steps": [
      {
        "kind": "A String",
        "name": "A String",
        "properties": {
          "a_key": "",
        },
      },
    ],
    "currentStateTime": "A String",
    "type": "A String",
    "id": "A String",
    "currentState": "A String",
    "executionInfo": {
      "stages": {
        "a_key": {
          "stepName": [
            "A String",
          ],
        },
      },
    },
  }
get(projectId, jobId, view=None)
Gets the state of the specified dataflow job.

Args:
  projectId: string, A parameter (required)
  jobId: string, A parameter (required)
  view: string, A parameter
    Allowed values
      JOB_VIEW_ALL - 
      JOB_VIEW_SUMMARY - 
      JOB_VIEW_UNKNOWN - 

Returns:
  An object of the form:

    {
    "requestedState": "A String",
    "name": "A String",
    "projectId": "A String",
    "createTime": "A String",
    "environment": {
      "version": {
        "a_key": "",
      },
      "workerPools": [
        {
          "diskSourceImage": "A String",
          "teardownPolicy": "A String",
          "kind": "A String",
          "packages": [
            {
              "location": "A String",
              "name": "A String",
            },
          ],
          "machineType": "A String",
          "zone": "A String",
          "onHostMaintenance": "A String",
          "taskrunnerSettings": {
            "workflowFileName": "A String",
            "logUploadLocation": "A String",
            "commandlinesFileName": "A String",
            "alsologtostderr": True or False,
            "vmId": "A String",
            "baseTaskDir": "A String",
            "continueOnException": True or False,
            "baseUrl": "A String",
            "parallelWorkerSettings": {
              "reportingEnabled": True or False,
              "shuffleServicePath": "A String",
              "workerId": "A String",
              "baseUrl": "A String",
              "servicePath": "A String",
              "tempStoragePrefix": "A String",
            },
            "oauthScopes": [
              "A String",
            ],
            "taskGroup": "A String",
            "languageHint": "A String",
            "logToSerialconsole": True or False,
            "streamingWorkerMainClass": "A String",
            "logDir": "A String",
            "dataflowApiVersion": "A String",
            "taskUser": "A String",
            "harnessCommand": "A String",
            "tempStoragePrefix": "A String",
          },
          "diskSizeGb": 42,
          "numWorkers": 42,
          "defaultPackageSet": "A String",
          "metadata": {
            "a_key": "A String",
          },
          "autoscalingSettings": {
            "maxNumWorkers": 42,
            "algorithm": "A String",
          },
          "dataDisks": [
            {
              "mountPoint": "A String",
              "sizeGb": 42,
              "diskType": "A String",
            },
          ],
        },
      ],
      "dataset": "A String",
      "experiments": [
        "A String",
      ],
      "sdkPipelineOptions": {
        "a_key": "",
      },
      "userAgent": {
        "a_key": "",
      },
      "clusterManagerApiService": "A String",
      "tempStoragePrefix": "A String",
    },
    "steps": [
      {
        "kind": "A String",
        "name": "A String",
        "properties": {
          "a_key": "",
        },
      },
    ],
    "currentStateTime": "A String",
    "type": "A String",
    "id": "A String",
    "currentState": "A String",
    "executionInfo": {
      "stages": {
        "a_key": {
          "stepName": [
            "A String",
          ],
        },
      },
    },
  }
getMetrics(projectId, jobId, startTime=None)
Request the job status.

Args:
  projectId: string, A parameter (required)
  jobId: string, A parameter (required)
  startTime: string, A parameter

Returns:
  An object of the form:

    {
    "metrics": [
      {
        "meanCount": "",
        "kind": "A String",
        "set": "",
        "name": {
          "origin": "A String",
          "name": "A String",
          "context": {
            "a_key": "A String",
          },
        },
        "meanSum": "",
        "cumulative": True or False,
        "updateTime": "A String",
        "scalar": "",
        "internal": "",
      },
    ],
    "metricTime": "A String",
  }
list(projectId, pageToken=None, pageSize=None, view=None)
List the jobs of a project

Args:
  projectId: string, A parameter (required)
  pageToken: string, A parameter
  pageSize: integer, A parameter
  view: string, A parameter
    Allowed values
      JOB_VIEW_ALL - 
      JOB_VIEW_SUMMARY - 
      JOB_VIEW_UNKNOWN - 

Returns:
  An object of the form:

    {
    "nextPageToken": "A String",
    "jobs": [
      {
        "requestedState": "A String",
        "name": "A String",
        "projectId": "A String",
        "createTime": "A String",
        "environment": {
          "version": {
            "a_key": "",
          },
          "workerPools": [
            {
              "diskSourceImage": "A String",
              "teardownPolicy": "A String",
              "kind": "A String",
              "packages": [
                {
                  "location": "A String",
                  "name": "A String",
                },
              ],
              "machineType": "A String",
              "zone": "A String",
              "onHostMaintenance": "A String",
              "taskrunnerSettings": {
                "workflowFileName": "A String",
                "logUploadLocation": "A String",
                "commandlinesFileName": "A String",
                "alsologtostderr": True or False,
                "vmId": "A String",
                "baseTaskDir": "A String",
                "continueOnException": True or False,
                "baseUrl": "A String",
                "parallelWorkerSettings": {
                  "reportingEnabled": True or False,
                  "shuffleServicePath": "A String",
                  "workerId": "A String",
                  "baseUrl": "A String",
                  "servicePath": "A String",
                  "tempStoragePrefix": "A String",
                },
                "oauthScopes": [
                  "A String",
                ],
                "taskGroup": "A String",
                "languageHint": "A String",
                "logToSerialconsole": True or False,
                "streamingWorkerMainClass": "A String",
                "logDir": "A String",
                "dataflowApiVersion": "A String",
                "taskUser": "A String",
                "harnessCommand": "A String",
                "tempStoragePrefix": "A String",
              },
              "diskSizeGb": 42,
              "numWorkers": 42,
              "defaultPackageSet": "A String",
              "metadata": {
                "a_key": "A String",
              },
              "autoscalingSettings": {
                "maxNumWorkers": 42,
                "algorithm": "A String",
              },
              "dataDisks": [
                {
                  "mountPoint": "A String",
                  "sizeGb": 42,
                  "diskType": "A String",
                },
              ],
            },
          ],
          "dataset": "A String",
          "experiments": [
            "A String",
          ],
          "sdkPipelineOptions": {
            "a_key": "",
          },
          "userAgent": {
            "a_key": "",
          },
          "clusterManagerApiService": "A String",
          "tempStoragePrefix": "A String",
        },
        "steps": [
          {
            "kind": "A String",
            "name": "A String",
            "properties": {
              "a_key": "",
            },
          },
        ],
        "currentStateTime": "A String",
        "type": "A String",
        "id": "A String",
        "currentState": "A String",
        "executionInfo": {
          "stages": {
            "a_key": {
              "stepName": [
                "A String",
              ],
            },
          },
        },
      },
    ],
  }
list_next(previous_request, previous_response)
Retrieves the next page of results.

Args:
  previous_request: The request for the previous page. (required)
  previous_response: The response from the request for the previous page. (required)

Returns:
  A request object that you can call 'execute()' on to request the next
  page. Returns None if there are no more items in the collection.
    
patch(projectId, jobId, body)
Updates the state of an existing dataflow job. This method supports patch semantics.

Args:
  projectId: string, A parameter (required)
  jobId: string, A parameter (required)
  body: object, The request body. (required)
    The object takes the form of:

{
  "requestedState": "A String",
  "name": "A String",
  "projectId": "A String",
  "createTime": "A String",
  "environment": {
    "version": {
      "a_key": "",
    },
    "workerPools": [
      {
        "diskSourceImage": "A String",
        "teardownPolicy": "A String",
        "kind": "A String",
        "packages": [
          {
            "location": "A String",
            "name": "A String",
          },
        ],
        "machineType": "A String",
        "zone": "A String",
        "onHostMaintenance": "A String",
        "taskrunnerSettings": {
          "workflowFileName": "A String",
          "logUploadLocation": "A String",
          "commandlinesFileName": "A String",
          "alsologtostderr": True or False,
          "vmId": "A String",
          "baseTaskDir": "A String",
          "continueOnException": True or False,
          "baseUrl": "A String",
          "parallelWorkerSettings": {
            "reportingEnabled": True or False,
            "shuffleServicePath": "A String",
            "workerId": "A String",
            "baseUrl": "A String",
            "servicePath": "A String",
            "tempStoragePrefix": "A String",
          },
          "oauthScopes": [
            "A String",
          ],
          "taskGroup": "A String",
          "languageHint": "A String",
          "logToSerialconsole": True or False,
          "streamingWorkerMainClass": "A String",
          "logDir": "A String",
          "dataflowApiVersion": "A String",
          "taskUser": "A String",
          "harnessCommand": "A String",
          "tempStoragePrefix": "A String",
        },
        "diskSizeGb": 42,
        "numWorkers": 42,
        "defaultPackageSet": "A String",
        "metadata": {
          "a_key": "A String",
        },
        "autoscalingSettings": {
          "maxNumWorkers": 42,
          "algorithm": "A String",
        },
        "dataDisks": [
          {
            "mountPoint": "A String",
            "sizeGb": 42,
            "diskType": "A String",
          },
        ],
      },
    ],
    "dataset": "A String",
    "experiments": [
      "A String",
    ],
    "sdkPipelineOptions": {
      "a_key": "",
    },
    "userAgent": {
      "a_key": "",
    },
    "clusterManagerApiService": "A String",
    "tempStoragePrefix": "A String",
  },
  "steps": [
    {
      "kind": "A String",
      "name": "A String",
      "properties": {
        "a_key": "",
      },
    },
  ],
  "currentStateTime": "A String",
  "type": "A String",
  "id": "A String",
  "currentState": "A String",
  "executionInfo": {
    "stages": {
      "a_key": {
        "stepName": [
          "A String",
        ],
      },
    },
  },
}


Returns:
  An object of the form:

    {
    "requestedState": "A String",
    "name": "A String",
    "projectId": "A String",
    "createTime": "A String",
    "environment": {
      "version": {
        "a_key": "",
      },
      "workerPools": [
        {
          "diskSourceImage": "A String",
          "teardownPolicy": "A String",
          "kind": "A String",
          "packages": [
            {
              "location": "A String",
              "name": "A String",
            },
          ],
          "machineType": "A String",
          "zone": "A String",
          "onHostMaintenance": "A String",
          "taskrunnerSettings": {
            "workflowFileName": "A String",
            "logUploadLocation": "A String",
            "commandlinesFileName": "A String",
            "alsologtostderr": True or False,
            "vmId": "A String",
            "baseTaskDir": "A String",
            "continueOnException": True or False,
            "baseUrl": "A String",
            "parallelWorkerSettings": {
              "reportingEnabled": True or False,
              "shuffleServicePath": "A String",
              "workerId": "A String",
              "baseUrl": "A String",
              "servicePath": "A String",
              "tempStoragePrefix": "A String",
            },
            "oauthScopes": [
              "A String",
            ],
            "taskGroup": "A String",
            "languageHint": "A String",
            "logToSerialconsole": True or False,
            "streamingWorkerMainClass": "A String",
            "logDir": "A String",
            "dataflowApiVersion": "A String",
            "taskUser": "A String",
            "harnessCommand": "A String",
            "tempStoragePrefix": "A String",
          },
          "diskSizeGb": 42,
          "numWorkers": 42,
          "defaultPackageSet": "A String",
          "metadata": {
            "a_key": "A String",
          },
          "autoscalingSettings": {
            "maxNumWorkers": 42,
            "algorithm": "A String",
          },
          "dataDisks": [
            {
              "mountPoint": "A String",
              "sizeGb": 42,
              "diskType": "A String",
            },
          ],
        },
      ],
      "dataset": "A String",
      "experiments": [
        "A String",
      ],
      "sdkPipelineOptions": {
        "a_key": "",
      },
      "userAgent": {
        "a_key": "",
      },
      "clusterManagerApiService": "A String",
      "tempStoragePrefix": "A String",
    },
    "steps": [
      {
        "kind": "A String",
        "name": "A String",
        "properties": {
          "a_key": "",
        },
      },
    ],
    "currentStateTime": "A String",
    "type": "A String",
    "id": "A String",
    "currentState": "A String",
    "executionInfo": {
      "stages": {
        "a_key": {
          "stepName": [
            "A String",
          ],
        },
      },
    },
  }
update(projectId, jobId, body)
Updates the state of an existing dataflow job.

Args:
  projectId: string, A parameter (required)
  jobId: string, A parameter (required)
  body: object, The request body. (required)
    The object takes the form of:

{
  "requestedState": "A String",
  "name": "A String",
  "projectId": "A String",
  "createTime": "A String",
  "environment": {
    "version": {
      "a_key": "",
    },
    "workerPools": [
      {
        "diskSourceImage": "A String",
        "teardownPolicy": "A String",
        "kind": "A String",
        "packages": [
          {
            "location": "A String",
            "name": "A String",
          },
        ],
        "machineType": "A String",
        "zone": "A String",
        "onHostMaintenance": "A String",
        "taskrunnerSettings": {
          "workflowFileName": "A String",
          "logUploadLocation": "A String",
          "commandlinesFileName": "A String",
          "alsologtostderr": True or False,
          "vmId": "A String",
          "baseTaskDir": "A String",
          "continueOnException": True or False,
          "baseUrl": "A String",
          "parallelWorkerSettings": {
            "reportingEnabled": True or False,
            "shuffleServicePath": "A String",
            "workerId": "A String",
            "baseUrl": "A String",
            "servicePath": "A String",
            "tempStoragePrefix": "A String",
          },
          "oauthScopes": [
            "A String",
          ],
          "taskGroup": "A String",
          "languageHint": "A String",
          "logToSerialconsole": True or False,
          "streamingWorkerMainClass": "A String",
          "logDir": "A String",
          "dataflowApiVersion": "A String",
          "taskUser": "A String",
          "harnessCommand": "A String",
          "tempStoragePrefix": "A String",
        },
        "diskSizeGb": 42,
        "numWorkers": 42,
        "defaultPackageSet": "A String",
        "metadata": {
          "a_key": "A String",
        },
        "autoscalingSettings": {
          "maxNumWorkers": 42,
          "algorithm": "A String",
        },
        "dataDisks": [
          {
            "mountPoint": "A String",
            "sizeGb": 42,
            "diskType": "A String",
          },
        ],
      },
    ],
    "dataset": "A String",
    "experiments": [
      "A String",
    ],
    "sdkPipelineOptions": {
      "a_key": "",
    },
    "userAgent": {
      "a_key": "",
    },
    "clusterManagerApiService": "A String",
    "tempStoragePrefix": "A String",
  },
  "steps": [
    {
      "kind": "A String",
      "name": "A String",
      "properties": {
        "a_key": "",
      },
    },
  ],
  "currentStateTime": "A String",
  "type": "A String",
  "id": "A String",
  "currentState": "A String",
  "executionInfo": {
    "stages": {
      "a_key": {
        "stepName": [
          "A String",
        ],
      },
    },
  },
}


Returns:
  An object of the form:

    {
    "requestedState": "A String",
    "name": "A String",
    "projectId": "A String",
    "createTime": "A String",
    "environment": {
      "version": {
        "a_key": "",
      },
      "workerPools": [
        {
          "diskSourceImage": "A String",
          "teardownPolicy": "A String",
          "kind": "A String",
          "packages": [
            {
              "location": "A String",
              "name": "A String",
            },
          ],
          "machineType": "A String",
          "zone": "A String",
          "onHostMaintenance": "A String",
          "taskrunnerSettings": {
            "workflowFileName": "A String",
            "logUploadLocation": "A String",
            "commandlinesFileName": "A String",
            "alsologtostderr": True or False,
            "vmId": "A String",
            "baseTaskDir": "A String",
            "continueOnException": True or False,
            "baseUrl": "A String",
            "parallelWorkerSettings": {
              "reportingEnabled": True or False,
              "shuffleServicePath": "A String",
              "workerId": "A String",
              "baseUrl": "A String",
              "servicePath": "A String",
              "tempStoragePrefix": "A String",
            },
            "oauthScopes": [
              "A String",
            ],
            "taskGroup": "A String",
            "languageHint": "A String",
            "logToSerialconsole": True or False,
            "streamingWorkerMainClass": "A String",
            "logDir": "A String",
            "dataflowApiVersion": "A String",
            "taskUser": "A String",
            "harnessCommand": "A String",
            "tempStoragePrefix": "A String",
          },
          "diskSizeGb": 42,
          "numWorkers": 42,
          "defaultPackageSet": "A String",
          "metadata": {
            "a_key": "A String",
          },
          "autoscalingSettings": {
            "maxNumWorkers": 42,
            "algorithm": "A String",
          },
          "dataDisks": [
            {
              "mountPoint": "A String",
              "sizeGb": 42,
              "diskType": "A String",
            },
          ],
        },
      ],
      "dataset": "A String",
      "experiments": [
        "A String",
      ],
      "sdkPipelineOptions": {
        "a_key": "",
      },
      "userAgent": {
        "a_key": "",
      },
      "clusterManagerApiService": "A String",
      "tempStoragePrefix": "A String",
    },
    "steps": [
      {
        "kind": "A String",
        "name": "A String",
        "properties": {
          "a_key": "",
        },
      },
    ],
    "currentStateTime": "A String",
    "type": "A String",
    "id": "A String",
    "currentState": "A String",
    "executionInfo": {
      "stages": {
        "a_key": {
          "stepName": [
            "A String",
          ],
        },
      },
    },
  }