Leases a dataflow WorkItem to run.
reportStatus(projectId, jobId, body)
Reports the status of dataflow WorkItems leased by a worker.
lease(projectId, jobId, body)
Leases a dataflow WorkItem to run. Args: projectId: string, A parameter (required) jobId: string, A parameter (required) body: object, The request body. (required) The object takes the form of: { "requestedLeaseDuration": "A String", "workItemTypes": [ "A String", ], "workerCapabilities": [ "A String", ], "currentWorkerTime": "A String", "workerId": "A String", } Returns: An object of the form: { "workItems": [ { "configuration": "A String", "leaseExpireTime": "A String", "seqMapTask": { "inputs": [ { "sources": [ { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, ], "kind": { "a_key": "", }, "tag": "A String", }, ], "name": "A String", "stageName": "A String", "systemName": "A String", "userFn": { "a_key": "", }, "outputInfos": [ { "tag": "A String", "sink": { "codec": { "a_key": "", }, "spec": { "a_key": "", }, }, }, ], }, "projectId": "A String", "sourceOperationTask": { "getMetadata": { "source": { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, }, "split": { "source": { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, "options": { "desiredShardSizeBytes": "A String", }, }, }, "mapTask": { "systemName": "A String", "stageName": "A String", "instructions": [ { "name": "A String", "read": { "source": { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, }, "outputs": [ { "codec": { "a_key": "", }, "name": "A String", }, ], "partialGroupByKey": { "input": { "outputNum": 42, "producerInstructionIndex": 42, }, "inputElementCodec": { "a_key": "", }, }, "write": { "input": { "outputNum": 42, "producerInstructionIndex": 42, }, "sink": { "codec": { "a_key": "", }, "spec": { "a_key": "", }, }, }, "systemName": "A String", "flatten": { "inputs": [ { "outputNum": 42, "producerInstructionIndex": 42, }, ], }, "parDo": { "sideInputs": [ { "sources": [ { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, ], "kind": { "a_key": "", }, "tag": "A String", }, ], "input": { "outputNum": 42, "producerInstructionIndex": 42, }, "multiOutputInfos": [ { "tag": "A String", }, ], "numOutputs": 42, "userFn": { "a_key": "", }, }, }, ], }, "jobId": "A String", "reportStatusInterval": "A String", "streamingSetupTask": { "workerHarnessPort": 42, "streamingComputationTopology": { "computations": [ { "inputs": [ { "streamingStageLocation": { "streamId": "A String", }, "pubsubLocation": { "idLabel": "A String", "topic": "A String", "subscription": "A String", "timestampLabel": "A String", "dropLateData": True or False, }, }, ], "computationId": "A String", "keyRanges": [ { "start": "A String", "deliveryEndpoint": "A String", "end": "A String", "persistentDirectory": "A String", "dataDisk": "A String", }, ], "outputs": [ { "streamingStageLocation": { "streamId": "A String", }, "pubsubLocation": { "idLabel": "A String", "topic": "A String", "subscription": "A String", "timestampLabel": "A String", "dropLateData": True or False, }, }, ], }, ], "dataDiskAssignments": [ { "vmInstance": "A String", "dataDisks": [ "A String", ], }, ], }, "receiveWorkPort": 42, }, "packages": [ { "location": "A String", "name": "A String", }, ], "shellTask": { "command": "A String", "exitCode": 42, }, "id": "A String", }, ], }
reportStatus(projectId, jobId, body)
Reports the status of dataflow WorkItems leased by a worker. Args: projectId: string, A parameter (required) jobId: string, A parameter (required) body: object, The request body. (required) The object takes the form of: { "workerId": "A String", "currentWorkerTime": "A String", "workItemStatuses": [ { "reportIndex": "A String", "errors": [ { "message": "A String", "code": 42, "details": [ { "a_key": "", }, ], }, ], "sourceOperationResponse": { "getMetadata": { "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, "split": { "outcome": "A String", "shards": [ { "derivationMode": "A String", "source": { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, }, ], }, }, "stopPosition": { "recordIndex": "A String", "end": True or False, "byteOffset": "A String", "key": "A String", "shufflePosition": "A String", }, "sourceFork": { "primary": { "derivationMode": "A String", "source": { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, }, "residual": { "derivationMode": "A String", "source": { "codec": { "a_key": "", }, "baseSpecs": [ { "a_key": "", }, ], "spec": { "a_key": "", }, "doesNotNeedSplitting": True or False, "metadata": { "infinite": True or False, "estimatedSizeBytes": "A String", "producesSortedKeys": True or False, }, }, }, }, "requestedLeaseDuration": "A String", "completed": True or False, "workItemId": "A String", "progress": { "position": { "recordIndex": "A String", "end": True or False, "byteOffset": "A String", "key": "A String", "shufflePosition": "A String", }, "remainingTime": "A String", "percentComplete": 3.14, }, "metricUpdates": [ { "meanCount": "", "kind": "A String", "set": "", "name": { "origin": "A String", "name": "A String", "context": { "a_key": "A String", }, }, "meanSum": "", "cumulative": True or False, "updateTime": "A String", "scalar": "", "internal": "", }, ], }, ], } Returns: An object of the form: { "workItemServiceStates": [ { "suggestedStopPoint": { "position": { "recordIndex": "A String", "end": True or False, "byteOffset": "A String", "key": "A String", "shufflePosition": "A String", }, "remainingTime": "A String", "percentComplete": 3.14, }, "leaseExpireTime": "A String", "suggestedStopPosition": { "recordIndex": "A String", "end": True or False, "byteOffset": "A String", "key": "A String", "shufflePosition": "A String", }, "reportStatusInterval": "A String", "harnessData": { "a_key": "", }, }, ], }