Google Dataflow API . v1b3 . projects . jobs . workItems

Instance Methods

lease(projectId, jobId, body)

Leases a dataflow WorkItem to run.

reportStatus(projectId, jobId, body)

Reports the status of dataflow WorkItems leased by a worker.

Method Details

lease(projectId, jobId, body)
Leases a dataflow WorkItem to run.

Args:
  projectId: string, A parameter (required)
  jobId: string, A parameter (required)
  body: object, The request body. (required)
    The object takes the form of:

{
    "requestedLeaseDuration": "A String",
    "workItemTypes": [
      "A String",
    ],
    "workerCapabilities": [
      "A String",
    ],
    "currentWorkerTime": "A String",
    "workerId": "A String",
  }


Returns:
  An object of the form:

    {
    "workItems": [
      {
        "configuration": "A String",
        "leaseExpireTime": "A String",
        "seqMapTask": {
          "inputs": [
            {
              "sources": [
                {
                  "codec": {
                    "a_key": "",
                  },
                  "baseSpecs": [
                    {
                      "a_key": "",
                    },
                  ],
                  "spec": {
                    "a_key": "",
                  },
                  "doesNotNeedSplitting": True or False,
                  "metadata": {
                    "infinite": True or False,
                    "estimatedSizeBytes": "A String",
                    "producesSortedKeys": True or False,
                  },
                },
              ],
              "kind": {
                "a_key": "",
              },
              "tag": "A String",
            },
          ],
          "name": "A String",
          "stageName": "A String",
          "systemName": "A String",
          "userFn": {
            "a_key": "",
          },
          "outputInfos": [
            {
              "tag": "A String",
              "sink": {
                "codec": {
                  "a_key": "",
                },
                "spec": {
                  "a_key": "",
                },
              },
            },
          ],
        },
        "projectId": "A String",
        "sourceOperationTask": {
          "getMetadata": {
            "source": {
              "codec": {
                "a_key": "",
              },
              "baseSpecs": [
                {
                  "a_key": "",
                },
              ],
              "spec": {
                "a_key": "",
              },
              "doesNotNeedSplitting": True or False,
              "metadata": {
                "infinite": True or False,
                "estimatedSizeBytes": "A String",
                "producesSortedKeys": True or False,
              },
            },
          },
          "split": {
            "source": {
              "codec": {
                "a_key": "",
              },
              "baseSpecs": [
                {
                  "a_key": "",
                },
              ],
              "spec": {
                "a_key": "",
              },
              "doesNotNeedSplitting": True or False,
              "metadata": {
                "infinite": True or False,
                "estimatedSizeBytes": "A String",
                "producesSortedKeys": True or False,
              },
            },
            "options": {
              "desiredShardSizeBytes": "A String",
            },
          },
        },
        "mapTask": {
          "systemName": "A String",
          "stageName": "A String",
          "instructions": [
            {
              "name": "A String",
              "read": {
                "source": {
                  "codec": {
                    "a_key": "",
                  },
                  "baseSpecs": [
                    {
                      "a_key": "",
                    },
                  ],
                  "spec": {
                    "a_key": "",
                  },
                  "doesNotNeedSplitting": True or False,
                  "metadata": {
                    "infinite": True or False,
                    "estimatedSizeBytes": "A String",
                    "producesSortedKeys": True or False,
                  },
                },
              },
              "outputs": [
                {
                  "codec": {
                    "a_key": "",
                  },
                  "name": "A String",
                },
              ],
              "partialGroupByKey": {
                "input": {
                  "outputNum": 42,
                  "producerInstructionIndex": 42,
                },
                "inputElementCodec": {
                  "a_key": "",
                },
              },
              "write": {
                "input": {
                  "outputNum": 42,
                  "producerInstructionIndex": 42,
                },
                "sink": {
                  "codec": {
                    "a_key": "",
                  },
                  "spec": {
                    "a_key": "",
                  },
                },
              },
              "systemName": "A String",
              "flatten": {
                "inputs": [
                  {
                    "outputNum": 42,
                    "producerInstructionIndex": 42,
                  },
                ],
              },
              "parDo": {
                "sideInputs": [
                  {
                    "sources": [
                      {
                        "codec": {
                          "a_key": "",
                        },
                        "baseSpecs": [
                          {
                            "a_key": "",
                          },
                        ],
                        "spec": {
                          "a_key": "",
                        },
                        "doesNotNeedSplitting": True or False,
                        "metadata": {
                          "infinite": True or False,
                          "estimatedSizeBytes": "A String",
                          "producesSortedKeys": True or False,
                        },
                      },
                    ],
                    "kind": {
                      "a_key": "",
                    },
                    "tag": "A String",
                  },
                ],
                "input": {
                  "outputNum": 42,
                  "producerInstructionIndex": 42,
                },
                "multiOutputInfos": [
                  {
                    "tag": "A String",
                  },
                ],
                "numOutputs": 42,
                "userFn": {
                  "a_key": "",
                },
              },
            },
          ],
        },
        "jobId": "A String",
        "reportStatusInterval": "A String",
        "streamingSetupTask": {
          "workerHarnessPort": 42,
          "streamingComputationTopology": {
            "computations": [
              {
                "inputs": [
                  {
                    "streamingStageLocation": {
                      "streamId": "A String",
                    },
                    "pubsubLocation": {
                      "idLabel": "A String",
                      "topic": "A String",
                      "subscription": "A String",
                      "timestampLabel": "A String",
                      "dropLateData": True or False,
                    },
                  },
                ],
                "computationId": "A String",
                "keyRanges": [
                  {
                    "start": "A String",
                    "deliveryEndpoint": "A String",
                    "end": "A String",
                    "persistentDirectory": "A String",
                    "dataDisk": "A String",
                  },
                ],
                "outputs": [
                  {
                    "streamingStageLocation": {
                      "streamId": "A String",
                    },
                    "pubsubLocation": {
                      "idLabel": "A String",
                      "topic": "A String",
                      "subscription": "A String",
                      "timestampLabel": "A String",
                      "dropLateData": True or False,
                    },
                  },
                ],
              },
            ],
            "dataDiskAssignments": [
              {
                "vmInstance": "A String",
                "dataDisks": [
                  "A String",
                ],
              },
            ],
          },
          "receiveWorkPort": 42,
        },
        "packages": [
          {
            "location": "A String",
            "name": "A String",
          },
        ],
        "shellTask": {
          "command": "A String",
          "exitCode": 42,
        },
        "id": "A String",
      },
    ],
  }
reportStatus(projectId, jobId, body)
Reports the status of dataflow WorkItems leased by a worker.

Args:
  projectId: string, A parameter (required)
  jobId: string, A parameter (required)
  body: object, The request body. (required)
    The object takes the form of:

{
    "workerId": "A String",
    "currentWorkerTime": "A String",
    "workItemStatuses": [
      {
        "reportIndex": "A String",
        "errors": [
          {
            "message": "A String",
            "code": 42,
            "details": [
              {
                "a_key": "",
              },
            ],
          },
        ],
        "sourceOperationResponse": {
          "getMetadata": {
            "metadata": {
              "infinite": True or False,
              "estimatedSizeBytes": "A String",
              "producesSortedKeys": True or False,
            },
          },
          "split": {
            "outcome": "A String",
            "shards": [
              {
                "derivationMode": "A String",
                "source": {
                  "codec": {
                    "a_key": "",
                  },
                  "baseSpecs": [
                    {
                      "a_key": "",
                    },
                  ],
                  "spec": {
                    "a_key": "",
                  },
                  "doesNotNeedSplitting": True or False,
                  "metadata": {
                    "infinite": True or False,
                    "estimatedSizeBytes": "A String",
                    "producesSortedKeys": True or False,
                  },
                },
              },
            ],
          },
        },
        "stopPosition": {
          "recordIndex": "A String",
          "end": True or False,
          "byteOffset": "A String",
          "key": "A String",
          "shufflePosition": "A String",
        },
        "sourceFork": {
          "primary": {
            "derivationMode": "A String",
            "source": {
              "codec": {
                "a_key": "",
              },
              "baseSpecs": [
                {
                  "a_key": "",
                },
              ],
              "spec": {
                "a_key": "",
              },
              "doesNotNeedSplitting": True or False,
              "metadata": {
                "infinite": True or False,
                "estimatedSizeBytes": "A String",
                "producesSortedKeys": True or False,
              },
            },
          },
          "residual": {
            "derivationMode": "A String",
            "source": {
              "codec": {
                "a_key": "",
              },
              "baseSpecs": [
                {
                  "a_key": "",
                },
              ],
              "spec": {
                "a_key": "",
              },
              "doesNotNeedSplitting": True or False,
              "metadata": {
                "infinite": True or False,
                "estimatedSizeBytes": "A String",
                "producesSortedKeys": True or False,
              },
            },
          },
        },
        "requestedLeaseDuration": "A String",
        "completed": True or False,
        "workItemId": "A String",
        "progress": {
          "position": {
            "recordIndex": "A String",
            "end": True or False,
            "byteOffset": "A String",
            "key": "A String",
            "shufflePosition": "A String",
          },
          "remainingTime": "A String",
          "percentComplete": 3.14,
        },
        "metricUpdates": [
          {
            "meanCount": "",
            "kind": "A String",
            "set": "",
            "name": {
              "origin": "A String",
              "name": "A String",
              "context": {
                "a_key": "A String",
              },
            },
            "meanSum": "",
            "cumulative": True or False,
            "updateTime": "A String",
            "scalar": "",
            "internal": "",
          },
        ],
      },
    ],
  }


Returns:
  An object of the form:

    {
    "workItemServiceStates": [
      {
        "suggestedStopPoint": {
          "position": {
            "recordIndex": "A String",
            "end": True or False,
            "byteOffset": "A String",
            "key": "A String",
            "shufflePosition": "A String",
          },
          "remainingTime": "A String",
          "percentComplete": 3.14,
        },
        "leaseExpireTime": "A String",
        "suggestedStopPosition": {
          "recordIndex": "A String",
          "end": True or False,
          "byteOffset": "A String",
          "key": "A String",
          "shufflePosition": "A String",
        },
        "reportStatusInterval": "A String",
        "harnessData": {
          "a_key": "",
        },
      },
    ],
  }