@@ -239,18 +239,62 @@ instance ToJSON SourceSplitResponse where
239239 ("shards" .=) <$> _ssrShards,
240240 ("outcome" .=) <$> _ssrOutcome])
241241
242−-- | Request to create a Dataflow job.
242+--
243+-- /See:/ 'resourceUtilizationReportMetricsItem' smart constructor.
244+newtype ResourceUtilizationReportMetricsItem = ResourceUtilizationReportMetricsItem'
245+ { _rurmiAddtional :: HashMap Text JSONValue
246+ } deriving (Eq,Show,Data,Typeable,Generic)
247+
248+-- | Creates a value of 'ResourceUtilizationReportMetricsItem' with the minimum fields required to make a request.
249+--
250+-- Use one of the following lenses to modify other fields as desired:
251+--
252+-- * 'rurmiAddtional'
253+resourceUtilizationReportMetricsItem
254+ :: HashMap Text JSONValue -- ^ 'rurmiAddtional'
255+ -> ResourceUtilizationReportMetricsItem
256+resourceUtilizationReportMetricsItem pRurmiAddtional_ =
257+ ResourceUtilizationReportMetricsItem'
258+ { _rurmiAddtional = _Coerce # pRurmiAddtional_
259+ }
260+
261+-- | Properties of the object.
262+rurmiAddtional :: Lens' ResourceUtilizationReportMetricsItem (HashMap Text JSONValue)
263+rurmiAddtional
264+ = lens _rurmiAddtional
265+ (\ s a -> s{_rurmiAddtional = a})
266+ . _Coerce
267+
268+instance FromJSON
269+ ResourceUtilizationReportMetricsItem where
270+ parseJSON
271+ = withObject "ResourceUtilizationReportMetricsItem"
272+ (\ o ->
273+ ResourceUtilizationReportMetricsItem' <$>
274+ (parseJSONObject o))
275+
276+instance ToJSON ResourceUtilizationReportMetricsItem
277+ where
278+ toJSON = toJSON . _rurmiAddtional
279+
280+-- | A request to create a Cloud Dataflow job from a template.
243281--
244282-- /See:/ 'createJobFromTemplateRequest' smart constructor.
245283data CreateJobFromTemplateRequest = CreateJobFromTemplateRequest'
246− { _cjftrGcsPath :: !(Maybe Text)
247− , _cjftrParameters :: !(Maybe CreateJobFromTemplateRequestParameters)
284+ { _cjftrEnvironment :: !(Maybe RuntimeEnvironment)
285+ , _cjftrJobName :: !(Maybe Text)
286+ , _cjftrGcsPath :: !(Maybe Text)
287+ , _cjftrParameters :: !(Maybe CreateJobFromTemplateRequestParameters)
248288 } deriving (Eq,Show,Data,Typeable,Generic)
249289
250290-- | Creates a value of 'CreateJobFromTemplateRequest' with the minimum fields required to make a request.
251291--
252292-- Use one of the following lenses to modify other fields as desired:
253293--
294+-- * 'cjftrEnvironment'
295+--
296+-- * 'cjftrJobName'
297+--
254298-- * 'cjftrGcsPath'
255299--
256300-- * 'cjftrParameters'
@@ -258,16 +302,30 @@ createJobFromTemplateRequest
258302 :: CreateJobFromTemplateRequest
259303createJobFromTemplateRequest =
260304 CreateJobFromTemplateRequest'
261− { _cjftrGcsPath = Nothing
305+ { _cjftrEnvironment = Nothing
306+ , _cjftrJobName = Nothing
307+ , _cjftrGcsPath = Nothing
262308 , _cjftrParameters = Nothing
263309 }
264310
265−-- | A path to the serialized JSON representation of the job.
311+-- | The runtime environment for the job.
312+cjftrEnvironment :: Lens' CreateJobFromTemplateRequest (Maybe RuntimeEnvironment)
313+cjftrEnvironment
314+ = lens _cjftrEnvironment
315+ (\ s a -> s{_cjftrEnvironment = a})
316+
317+-- | Required. The job name to use for the created job.
318+cjftrJobName :: Lens' CreateJobFromTemplateRequest (Maybe Text)
319+cjftrJobName
320+ = lens _cjftrJobName (\ s a -> s{_cjftrJobName = a})
321+
322+-- | Required. A Cloud Storage path to the template from which to create the
323+-- job. Must be a valid Cloud Storage URL, beginning with \`gs:\/\/\`.
266324cjftrGcsPath :: Lens' CreateJobFromTemplateRequest (Maybe Text)
267325cjftrGcsPath
268326 = lens _cjftrGcsPath (\ s a -> s{_cjftrGcsPath = a})
269327
270−-- | Dynamic parameterization of the job\'s runtime environment.
328+-- | The runtime parameters to pass to the job.
271329cjftrParameters :: Lens' CreateJobFromTemplateRequest (Maybe CreateJobFromTemplateRequestParameters)
272330cjftrParameters
273331 = lens _cjftrParameters
@@ -278,13 +336,17 @@ instance FromJSON CreateJobFromTemplateRequest where
278336 = withObject "CreateJobFromTemplateRequest"
279337 (\ o ->
280338 CreateJobFromTemplateRequest' <$>
281− (o .:? "gcsPath") <*> (o .:? "parameters"))
339+ (o .:? "environment") <*> (o .:? "jobName") <*>
340+ (o .:? "gcsPath")
341+ <*> (o .:? "parameters"))
282342
283343instance ToJSON CreateJobFromTemplateRequest where
284344 toJSON CreateJobFromTemplateRequest'{..}
285345 = object
286346 (catMaybes
287− [("gcsPath" .=) <$> _cjftrGcsPath,
347+ [("environment" .=) <$> _cjftrEnvironment,
348+ ("jobName" .=) <$> _cjftrJobName,
349+ ("gcsPath" .=) <$> _cjftrGcsPath,
288350 ("parameters" .=) <$> _cjftrParameters])
289351
290352-- | The \`Status\` type defines a logical error model that is suitable for
@@ -1031,6 +1093,7 @@ instance ToJSON InstructionOutput where
10311093-- /See:/ 'reportWorkItemStatusRequest' smart constructor.
10321094data ReportWorkItemStatusRequest = ReportWorkItemStatusRequest'
10331095 { _rwisrCurrentWorkerTime :: !(Maybe Text)
1096+ , _rwisrLocation :: !(Maybe Text)
10341097 , _rwisrWorkItemStatuses :: !(Maybe [WorkItemStatus])
10351098 , _rwisrWorkerId :: !(Maybe Text)
10361099 } deriving (Eq,Show,Data,Typeable,Generic)
@@ -1041,6 +1104,8 @@ data ReportWorkItemStatusRequest = ReportWorkItemStatusRequest'
10411104--
10421105-- * 'rwisrCurrentWorkerTime'
10431106--
1107+-- * 'rwisrLocation'
1108+--
10441109-- * 'rwisrWorkItemStatuses'
10451110--
10461111-- * 'rwisrWorkerId'
@@ -1049,6 +1114,7 @@ reportWorkItemStatusRequest
10491114reportWorkItemStatusRequest =
10501115 ReportWorkItemStatusRequest'
10511116 { _rwisrCurrentWorkerTime = Nothing
1117+ , _rwisrLocation = Nothing
10521118 , _rwisrWorkItemStatuses = Nothing
10531119 , _rwisrWorkerId = Nothing
10541120 }
@@ -1059,6 +1125,12 @@ rwisrCurrentWorkerTime
10591125 = lens _rwisrCurrentWorkerTime
10601126 (\ s a -> s{_rwisrCurrentWorkerTime = a})
10611127
1128+-- | The location which contains the WorkItem\'s job.
1129+rwisrLocation :: Lens' ReportWorkItemStatusRequest (Maybe Text)
1130+rwisrLocation
1131+ = lens _rwisrLocation
1132+ (\ s a -> s{_rwisrLocation = a})
1133+
10621134-- | The order is unimportant, except that the order of the
10631135-- WorkItemServiceState messages in the ReportWorkItemStatusResponse
10641136-- corresponds to the order of WorkItemStatus messages here.
@@ -1083,8 +1155,8 @@ instance FromJSON ReportWorkItemStatusRequest where
10831155 = withObject "ReportWorkItemStatusRequest"
10841156 (\ o ->
10851157 ReportWorkItemStatusRequest' <$>
1086− (o .:? "currentWorkerTime") <*>
1087− (o .:? "workItemStatuses" .!= mempty)
1158+ (o .:? "currentWorkerTime") <*> (o .:? "location")
1159+ <*> (o .:? "workItemStatuses" .!= mempty)
10881160 <*> (o .:? "workerId"))
10891161
10901162instance ToJSON ReportWorkItemStatusRequest where
@@ -1093,6 +1165,7 @@ instance ToJSON ReportWorkItemStatusRequest where
10931165 (catMaybes
10941166 [("currentWorkerTime" .=) <$>
10951167 _rwisrCurrentWorkerTime,
1168+ ("location" .=) <$> _rwisrLocation,
10961169 ("workItemStatuses" .=) <$> _rwisrWorkItemStatuses,
10971170 ("workerId" .=) <$> _rwisrWorkerId])
10981171
@@ -1616,12 +1689,12 @@ dSizeGb
16161689-- resource name typically ending in \"pd-standard\". If SSD persistent
16171690-- disks are available, the resource name typically ends with \"pd-ssd\".
16181691-- The actual valid values are defined the Google Compute Engine API, not
1619−-- by the Dataflow API; consult the Google Compute Engine documentation for
1620−-- more information about determining the set of available disk types for a
1621−-- particular project and zone. Google Compute Engine Disk types are local
1622−-- to a particular project in a particular zone, and so the resource name
1623−-- will typically look something like this:
1624−-- compute.googleapis.com\/projects\/ \/zones\/\/diskTypes\/pd-standard
1692+-- by the Cloud Dataflow API; consult the Google Compute Engine
1693+-- documentation for more information about determining the set of
1694+-- available disk types for a particular project and zone. Google Compute
1695+-- Engine Disk types are local to a particular project in a particular
1696+-- zone, and so the resource name will typically look something like this:
1697+-- compute.googleapis.com\/projects\/project-id\/zones\/zone\/diskTypes\/pd-standard
16251698dDiskType :: Lens' Disk (Maybe Text)
16261699dDiskType
16271700 = lens _dDiskType (\ s a -> s{_dDiskType = a})
@@ -1890,8 +1963,8 @@ eExperiments
18901963 . _Default
18911964 . _Coerce
18921965
1893−-- | Worker pools. At least one \"harness\" worker pool must be specified in
1894−-- order for the job to have workers.
1966+-- | The worker pools. At least one \"harness\" worker pool must be specified
1967+-- in order for the job to have workers.
18951968eWorkerPools :: Lens' Environment [WorkerPool]
18961969eWorkerPools
18971970 = lens _eWorkerPools (\ s a -> s{_eWorkerPools = a})
@@ -1942,10 +2015,10 @@ eUserAgent :: Lens' Environment (Maybe EnvironmentUserAgent)
19422015eUserAgent
19432016 = lens _eUserAgent (\ s a -> s{_eUserAgent = a})
19442017
1945−-- | The Dataflow SDK pipeline options specified by the user. These options
1946−-- are passed through the service and are used to recreate the SDK pipeline
1947−-- options on the worker in a language agnostic and platform independent
1948−-- way.
2018+-- | The Cloud Dataflow SDK pipeline options specified by the user. These
2019+-- options are passed through the service and are used to recreate the SDK
2020+-- pipeline options on the worker in a language agnostic and platform
2021+-- independent way.
19492022eSdkPipelineOptions :: Lens' Environment (Maybe EnvironmentSdkPipelineOptions)
19502023eSdkPipelineOptions
19512024 = lens _eSdkPipelineOptions
@@ -2082,6 +2155,7 @@ data CounterUpdate = CounterUpdate'
20822155 , _cuInternal :: !(Maybe JSONValue)
20832156 , _cuStringList :: !(Maybe StringList)
20842157 , _cuShortId :: !(Maybe (Textual Int64))
2158+ , _cuDistribution :: !(Maybe DistributionUpdate)
20852159 , _cuCumulative :: !(Maybe Bool)
20862160 , _cuStructuredNameAndMetadata :: !(Maybe CounterStructuredNameAndMetadata)
20872161 , _cuFloatingPointMean :: !(Maybe FloatingPointMean)
@@ -2107,6 +2181,8 @@ data CounterUpdate = CounterUpdate'
21072181--
21082182-- * 'cuShortId'
21092183--
2184+-- * 'cuDistribution'
2185+--
21102186-- * 'cuCumulative'
21112187--
21122188-- * 'cuStructuredNameAndMetadata'
@@ -2130,6 +2206,7 @@ counterUpdate =
21302206 , _cuInternal = Nothing
21312207 , _cuStringList = Nothing
21322208 , _cuShortId = Nothing
2209+ , _cuDistribution = Nothing
21332210 , _cuCumulative = Nothing
21342211 , _cuStructuredNameAndMetadata = Nothing
21352212 , _cuFloatingPointMean = Nothing
@@ -2175,6 +2252,12 @@ cuShortId
21752252 = lens _cuShortId (\ s a -> s{_cuShortId = a}) .
21762253 mapping _Coerce
21772254
2255+-- | Distribution data
2256+cuDistribution :: Lens' CounterUpdate (Maybe DistributionUpdate)
2257+cuDistribution
2258+ = lens _cuDistribution
2259+ (\ s a -> s{_cuDistribution = a})
2260+
21782261-- | True if this counter is reported as the total cumulative aggregate value
21792262-- accumulated since the worker started working on this WorkItem. By
21802263-- default this is false, indicating that this counter is reported as a
@@ -2227,6 +2310,7 @@ instance FromJSON CounterUpdate where
22272310 <*> (o .:? "internal")
22282311 <*> (o .:? "stringList")
22292312 <*> (o .:? "shortId")
2313+ <*> (o .:? "distribution")
22302314 <*> (o .:? "cumulative")
22312315 <*> (o .:? "structuredNameAndMetadata")
22322316 <*> (o .:? "floatingPointMean")
@@ -2245,6 +2329,7 @@ instance ToJSON CounterUpdate where
22452329 ("internal" .=) <$> _cuInternal,
22462330 ("stringList" .=) <$> _cuStringList,
22472331 ("shortId" .=) <$> _cuShortId,
2332+ ("distribution" .=) <$> _cuDistribution,
22482333 ("cumulative" .=) <$> _cuCumulative,
22492334 ("structuredNameAndMetadata" .=) <$>
22502335 _cuStructuredNameAndMetadata,
@@ -2820,7 +2905,7 @@ instance ToJSON WorkerMessageCode where
28202905 [("parameters" .=) <$> _wmcParameters,
28212906 ("code" .=) <$> _wmcCode])
28222907
2823−-- | Map of transform name prefixes of the job to be replaced to the
2908+-- | The map of transform name prefixes of the job to be replaced to the
28242909-- corresponding name prefixes of the new job.
28252910--
28262911-- /See:/ 'jobTransformNameMApping' smart constructor.
@@ -3085,19 +3170,19 @@ taskRunnerSettings =
30853170 , _trsLanguageHint = Nothing
30863171 }
30873172
3088−-- | Do we continue taskrunner if an exception is hit?
3173+-- | Whether to continue taskrunner if an exception is hit.
30893174trsContinueOnException :: Lens' TaskRunnerSettings (Maybe Bool)
30903175trsContinueOnException
30913176 = lens _trsContinueOnException
30923177 (\ s a -> s{_trsContinueOnException = a})
30933178
3094−-- | Command to launch the worker harness.
3179+-- | The command to launch the worker harness.
30953180trsHarnessCommand :: Lens' TaskRunnerSettings (Maybe Text)
30963181trsHarnessCommand
30973182 = lens _trsHarnessCommand
30983183 (\ s a -> s{_trsHarnessCommand = a})
30993184
3100−-- | Store the workflow in this file.
3185+-- | The file to store the workflow in.
31013186trsWorkflowFileName :: Lens' TaskRunnerSettings (Maybe Text)
31023187trsWorkflowFileName
31033188 = lens _trsWorkflowFileName
@@ -3109,30 +3194,30 @@ trsTaskGroup :: Lens' TaskRunnerSettings (Maybe Text)
31093194trsTaskGroup
31103195 = lens _trsTaskGroup (\ s a -> s{_trsTaskGroup = a})
31113196
3112−-- | Also send taskrunner log info to stderr?
3197+-- | Whether to also send taskrunner log info to stderr.
31133198trsAlsologtostderr :: Lens' TaskRunnerSettings (Maybe Bool)
31143199trsAlsologtostderr
31153200 = lens _trsAlsologtostderr
31163201 (\ s a -> s{_trsAlsologtostderr = a})
31173202
3118−-- | API version of endpoint, e.g. \"v1b3\"
3203+-- | The API version of endpoint, e.g. \"v1b3\"
31193204trsDataflowAPIVersion :: Lens' TaskRunnerSettings (Maybe Text)
31203205trsDataflowAPIVersion
31213206 = lens _trsDataflowAPIVersion
31223207 (\ s a -> s{_trsDataflowAPIVersion = a})
31233208
3124−-- | Directory on the VM to store logs.
3209+-- | The directory on the VM to store logs.
31253210trsLogDir :: Lens' TaskRunnerSettings (Maybe Text)
31263211trsLogDir
31273212 = lens _trsLogDir (\ s a -> s{_trsLogDir = a})
31283213
3129−-- | Store preprocessing commands in this file.
3214+-- | The file to store preprocessing commands in.
31303215trsCommandlinesFileName :: Lens' TaskRunnerSettings (Maybe Text)
31313216trsCommandlinesFileName
31323217 = lens _trsCommandlinesFileName
31333218 (\ s a -> s{_trsCommandlinesFileName = a})
31343219
3135−-- | ID string of VM.
3220+-- | The ID string of the VM.
31363221trsVMId :: Lens' TaskRunnerSettings (Maybe Text)
31373222trsVMId = lens _trsVMId (\ s a -> s{_trsVMId = a})
31383223
@@ -3146,8 +3231,8 @@ trsBaseURL :: Lens' TaskRunnerSettings (Maybe Text)
31463231trsBaseURL
31473232 = lens _trsBaseURL (\ s a -> s{_trsBaseURL = a})
31483233
3149−-- | OAuth2 scopes to be requested by the taskrunner in order to access the
3150−-- dataflow API.
3234+-- | The OAuth2 scopes to be requested by the taskrunner in order to access
3235+-- the Cloud Dataflow API.
31513236trsOAuthScopes :: Lens' TaskRunnerSettings [Text]
31523237trsOAuthScopes
31533238 = lens _trsOAuthScopes
@@ -3161,13 +3246,13 @@ trsTaskUser :: Lens' TaskRunnerSettings (Maybe Text)
31613246trsTaskUser
31623247 = lens _trsTaskUser (\ s a -> s{_trsTaskUser = a})
31633248
3164−-- | Streaming worker main class name.
3249+-- | The streaming worker main class name.
31653250trsStreamingWorkerMainClass :: Lens' TaskRunnerSettings (Maybe Text)
31663251trsStreamingWorkerMainClass
31673252 = lens _trsStreamingWorkerMainClass
31683253 (\ s a -> s{_trsStreamingWorkerMainClass = a})
31693254
3170−-- | Location on the worker for task-specific subdirectories.
3255+-- | The location on the worker for task-specific subdirectories.
31713256trsBaseTaskDir :: Lens' TaskRunnerSettings (Maybe Text)
31723257trsBaseTaskDir
31733258 = lens _trsBaseTaskDir
@@ -3191,19 +3276,20 @@ trsTempStoragePrefix
31913276 = lens _trsTempStoragePrefix
31923277 (\ s a -> s{_trsTempStoragePrefix = a})
31933278
3194−-- | Send taskrunner log into to Google Compute Engine VM serial console?
3279+-- | Whether to send taskrunner log info to Google Compute Engine VM serial
3280+-- console.
31953281trsLogToSerialconsole :: Lens' TaskRunnerSettings (Maybe Bool)
31963282trsLogToSerialconsole
31973283 = lens _trsLogToSerialconsole
31983284 (\ s a -> s{_trsLogToSerialconsole = a})
31993285
3200−-- | Settings to pass to the parallel worker harness.
3286+-- | The settings to pass to the parallel worker harness.
32013287trsParallelWorkerSettings :: Lens' TaskRunnerSettings (Maybe WorkerSettings)
32023288trsParallelWorkerSettings
32033289 = lens _trsParallelWorkerSettings
32043290 (\ s a -> s{_trsParallelWorkerSettings = a})
32053291
3206−-- | Suggested backend language.
3292+-- | The suggested backend language.
32073293trsLanguageHint :: Lens' TaskRunnerSettings (Maybe Text)
32083294trsLanguageHint
32093295 = lens _trsLanguageHint
@@ -3337,10 +3423,10 @@ instance ToJSON StreamingComputationConfig where
33373423 ("computationId" .=) <$> _sccComputationId,
33383424 ("stageName" .=) <$> _sccStageName])
33393425
3340−-- | The Dataflow SDK pipeline options specified by the user. These options
3341−-- are passed through the service and are used to recreate the SDK pipeline
3342−-- options on the worker in a language agnostic and platform independent
3343−-- way.
3426+-- | The Cloud Dataflow SDK pipeline options specified by the user. These
3427+-- options are passed through the service and are used to recreate the SDK
3428+-- pipeline options on the worker in a language agnostic and platform
3429+-- independent way.
33443430--
33453431-- /See:/ 'environmentSdkPipelineOptions' smart constructor.
33463432newtype EnvironmentSdkPipelineOptions = EnvironmentSdkPipelineOptions'
@@ -3429,6 +3515,7 @@ instance ToJSON FloatingPointMean where
34293515data LeaseWorkItemRequest = LeaseWorkItemRequest'
34303516 { _lwirWorkItemTypes :: !(Maybe [Text])
34313517 , _lwirCurrentWorkerTime :: !(Maybe Text)
3518+ , _lwirLocation :: !(Maybe Text)
34323519 , _lwirWorkerCapabilities :: !(Maybe [Text])
34333520 , _lwirRequestedLeaseDuration :: !(Maybe Text)
34343521 , _lwirWorkerId :: !(Maybe Text)
@@ -3442,6 +3529,8 @@ data LeaseWorkItemRequest = LeaseWorkItemRequest'
34423529--
34433530-- * 'lwirCurrentWorkerTime'
34443531--
3532+-- * 'lwirLocation'
3533+--
34453534-- * 'lwirWorkerCapabilities'
34463535--
34473536-- * 'lwirRequestedLeaseDuration'
@@ -3453,6 +3542,7 @@ leaseWorkItemRequest =
34533542 LeaseWorkItemRequest'
34543543 { _lwirWorkItemTypes = Nothing
34553544 , _lwirCurrentWorkerTime = Nothing
3545+ , _lwirLocation = Nothing
34563546 , _lwirWorkerCapabilities = Nothing
34573547 , _lwirRequestedLeaseDuration = Nothing
34583548 , _lwirWorkerId = Nothing
@@ -3472,6 +3562,11 @@ lwirCurrentWorkerTime
34723562 = lens _lwirCurrentWorkerTime
34733563 (\ s a -> s{_lwirCurrentWorkerTime = a})
34743564
3565+-- | The location which contains the WorkItem\'s job.
3566+lwirLocation :: Lens' LeaseWorkItemRequest (Maybe Text)
3567+lwirLocation
3568+ = lens _lwirLocation (\ s a -> s{_lwirLocation = a})
3569+
34753570-- | Worker capabilities. WorkItems might be limited to workers with specific
34763571-- capabilities.
34773572lwirWorkerCapabilities :: Lens' LeaseWorkItemRequest [Text]
@@ -3500,6 +3595,7 @@ instance FromJSON LeaseWorkItemRequest where
35003595 LeaseWorkItemRequest' <$>
35013596 (o .:? "workItemTypes" .!= mempty) <*>
35023597 (o .:? "currentWorkerTime")
3598+ <*> (o .:? "location")
35033599 <*> (o .:? "workerCapabilities" .!= mempty)
35043600 <*> (o .:? "requestedLeaseDuration")
35053601 <*> (o .:? "workerId"))
@@ -3510,6 +3606,7 @@ instance ToJSON LeaseWorkItemRequest where
35103606 (catMaybes
35113607 [("workItemTypes" .=) <$> _lwirWorkItemTypes,
35123608 ("currentWorkerTime" .=) <$> _lwirCurrentWorkerTime,
3609+ ("location" .=) <$> _lwirLocation,
35133610 ("workerCapabilities" .=) <$>
35143611 _lwirWorkerCapabilities,
35153612 ("requestedLeaseDuration" .=) <$>
@@ -4009,7 +4106,7 @@ workerSettings =
40094106 , _wsWorkerId = Nothing
40104107 }
40114108
4012−-- | The Dataflow service path relative to the root URL, for example,
4109+-- | The Cloud Dataflow service path relative to the root URL, for example,
40134110-- \"dataflow\/v1b3\/projects\".
40144111wsServicePath :: Lens' WorkerSettings (Maybe Text)
40154112wsServicePath
@@ -4042,13 +4139,13 @@ wsTempStoragePrefix
40424139 = lens _wsTempStoragePrefix
40434140 (\ s a -> s{_wsTempStoragePrefix = a})
40444141
4045−-- | Send work progress updates to service.
4142+-- | Whether to send work progress updates to the service.
40464143wsReportingEnabled :: Lens' WorkerSettings (Maybe Bool)
40474144wsReportingEnabled
40484145 = lens _wsReportingEnabled
40494146 (\ s a -> s{_wsReportingEnabled = a})
40504147
4051−-- | ID of the worker running this pipeline.
4148+-- | The ID of the worker running this pipeline.
40524149wsWorkerId :: Lens' WorkerSettings (Maybe Text)
40534150wsWorkerId
40544151 = lens _wsWorkerId (\ s a -> s{_wsWorkerId = a})
@@ -4202,12 +4299,86 @@ instance ToJSON DataDiskAssignment where
42024299 [("vmInstance" .=) <$> _ddaVMInstance,
42034300 ("dataDisks" .=) <$> _ddaDataDisks])
42044301
4302+-- | Worker metrics exported from workers. This contains resource utilization
4303+-- metrics accumulated from a variety of sources. For more information, see
4304+-- go\/df-resource-signals. Note that this proto closely follows the
4305+-- structure of its DFE siblings in its contents.
4306+--
4307+-- /See:/ 'resourceUtilizationReport' smart constructor.
4308+newtype ResourceUtilizationReport = ResourceUtilizationReport'
4309+ { _rurMetrics :: Maybe [ResourceUtilizationReportMetricsItem]
4310+ } deriving (Eq,Show,Data,Typeable,Generic)
4311+
4312+-- | Creates a value of 'ResourceUtilizationReport' with the minimum fields required to make a request.
4313+--
4314+-- Use one of the following lenses to modify other fields as desired:
4315+--
4316+-- * 'rurMetrics'
4317+resourceUtilizationReport
4318+ :: ResourceUtilizationReport
4319+resourceUtilizationReport =
4320+ ResourceUtilizationReport'
4321+ { _rurMetrics = Nothing
4322+ }
4323+
4324+-- | Each Struct must parallel DFE worker metrics protos (eg., cpu_time
4325+-- metric will have nested values “timestamp_ms, total_ms, rate”).
4326+rurMetrics :: Lens' ResourceUtilizationReport [ResourceUtilizationReportMetricsItem]
4327+rurMetrics
4328+ = lens _rurMetrics (\ s a -> s{_rurMetrics = a}) .
4329+ _Default
4330+ . _Coerce
4331+
4332+instance FromJSON ResourceUtilizationReport where
4333+ parseJSON
4334+ = withObject "ResourceUtilizationReport"
4335+ (\ o ->
4336+ ResourceUtilizationReport' <$>
4337+ (o .:? "metrics" .!= mempty))
4338+
4339+instance ToJSON ResourceUtilizationReport where
4340+ toJSON ResourceUtilizationReport'{..}
4341+ = object (catMaybes [("metrics" .=) <$> _rurMetrics])
4342+
4343+-- | Indicates which location failed to respond to a request for data.
4344+--
4345+-- /See:/ 'failedLocation' smart constructor.
4346+newtype FailedLocation = FailedLocation'
4347+ { _flName :: Maybe Text
4348+ } deriving (Eq,Show,Data,Typeable,Generic)
4349+
4350+-- | Creates a value of 'FailedLocation' with the minimum fields required to make a request.
4351+--
4352+-- Use one of the following lenses to modify other fields as desired:
4353+--
4354+-- * 'flName'
4355+failedLocation
4356+ :: FailedLocation
4357+failedLocation =
4358+ FailedLocation'
4359+ { _flName = Nothing
4360+ }
4361+
4362+-- | The name of the failed location.
4363+flName :: Lens' FailedLocation (Maybe Text)
4364+flName = lens _flName (\ s a -> s{_flName = a})
4365+
4366+instance FromJSON FailedLocation where
4367+ parseJSON
4368+ = withObject "FailedLocation"
4369+ (\ o -> FailedLocation' <$> (o .:? "name"))
4370+
4371+instance ToJSON FailedLocation where
4372+ toJSON FailedLocation'{..}
4373+ = object (catMaybes [("name" .=) <$> _flName])
4374+
42054375-- | A worker_message response allows the server to pass information to the
42064376-- sender.
42074377--
42084378-- /See:/ 'workerMessageResponse' smart constructor.
4209−newtype WorkerMessageResponse = WorkerMessageResponse'
4210− { _wmrWorkerHealthReportResponse :: Maybe WorkerHealthReportResponse
4379+data WorkerMessageResponse = WorkerMessageResponse'
4380+ { _wmrWorkerHealthReportResponse :: !(Maybe WorkerHealthReportResponse)
4381+ , _wmrWorkerMetricsResponse :: !(Maybe ResourceUtilizationReportResponse)
42114382 } deriving (Eq,Show,Data,Typeable,Generic)
42124383
42134384-- | Creates a value of 'WorkerMessageResponse' with the minimum fields required to make a request.
@@ -4215,11 +4386,14 @@ newtype WorkerMessageResponse = WorkerMessageResponse'
42154386-- Use one of the following lenses to modify other fields as desired:
42164387--
42174388-- * 'wmrWorkerHealthReportResponse'
4389+--
4390+-- * 'wmrWorkerMetricsResponse'
42184391workerMessageResponse
42194392 :: WorkerMessageResponse
42204393workerMessageResponse =
42214394 WorkerMessageResponse'
42224395 { _wmrWorkerHealthReportResponse = Nothing
4396+ , _wmrWorkerMetricsResponse = Nothing
42234397 }
42244398
42254399-- | The service\'s response to a worker\'s health report.
@@ -4228,21 +4402,30 @@ wmrWorkerHealthReportResponse
42284402 = lens _wmrWorkerHealthReportResponse
42294403 (\ s a -> s{_wmrWorkerHealthReportResponse = a})
42304404
4405+-- | Service\'s response to reporting worker metrics (currently empty).
4406+wmrWorkerMetricsResponse :: Lens' WorkerMessageResponse (Maybe ResourceUtilizationReportResponse)
4407+wmrWorkerMetricsResponse
4408+ = lens _wmrWorkerMetricsResponse
4409+ (\ s a -> s{_wmrWorkerMetricsResponse = a})
4410+
42314411instance FromJSON WorkerMessageResponse where
42324412 parseJSON
42334413 = withObject "WorkerMessageResponse"
42344414 (\ o ->
42354415 WorkerMessageResponse' <$>
4236− (o .:? "workerHealthReportResponse"))
4416+ (o .:? "workerHealthReportResponse") <*>
4417+ (o .:? "workerMetricsResponse"))
42374418
42384419instance ToJSON WorkerMessageResponse where
42394420 toJSON WorkerMessageResponse'{..}
42404421 = object
42414422 (catMaybes
42424423 [("workerHealthReportResponse" .=) <$>
4243− _wmrWorkerHealthReportResponse])
4424+ _wmrWorkerHealthReportResponse,
4425+ ("workerMetricsResponse" .=) <$>
4426+ _wmrWorkerMetricsResponse])
42444427
4245−-- | Dynamic parameterization of the job\'s runtime environment.
4428+-- | The runtime parameters to pass to the job.
42464429--
42474430-- /See:/ 'createJobFromTemplateRequestParameters' smart constructor.
42484431newtype CreateJobFromTemplateRequestParameters = CreateJobFromTemplateRequestParameters'
@@ -4932,6 +5115,96 @@ instance FromJSON WorkerPoolPoolArgs where
49325115instance ToJSON WorkerPoolPoolArgs where
49335116 toJSON = toJSON . _wppaAddtional
49345117
5118+-- | The environment values to set at runtime.
5119+--
5120+-- /See:/ 'runtimeEnvironment' smart constructor.
5121+data RuntimeEnvironment = RuntimeEnvironment'
5122+ { _reZone :: !(Maybe Text)
5123+ , _reBypassTempDirValidation :: !(Maybe Bool)
5124+ , _reServiceAccountEmail :: !(Maybe Text)
5125+ , _reMaxWorkers :: !(Maybe (Textual Int32))
5126+ , _reTempLocation :: !(Maybe Text)
5127+ } deriving (Eq,Show,Data,Typeable,Generic)
5128+
5129+-- | Creates a value of 'RuntimeEnvironment' with the minimum fields required to make a request.
5130+--
5131+-- Use one of the following lenses to modify other fields as desired:
5132+--
5133+-- * 'reZone'
5134+--
5135+-- * 'reBypassTempDirValidation'
5136+--
5137+-- * 'reServiceAccountEmail'
5138+--
5139+-- * 'reMaxWorkers'
5140+--
5141+-- * 'reTempLocation'
5142+runtimeEnvironment
5143+ :: RuntimeEnvironment
5144+runtimeEnvironment =
5145+ RuntimeEnvironment'
5146+ { _reZone = Nothing
5147+ , _reBypassTempDirValidation = Nothing
5148+ , _reServiceAccountEmail = Nothing
5149+ , _reMaxWorkers = Nothing
5150+ , _reTempLocation = Nothing
5151+ }
5152+
5153+-- | The Compute Engine [availability
5154+-- zone](https:\/\/cloud.google.com\/compute\/docs\/regions-zones\/regions-zones)
5155+-- for launching worker instances to run your pipeline.
5156+reZone :: Lens' RuntimeEnvironment (Maybe Text)
5157+reZone = lens _reZone (\ s a -> s{_reZone = a})
5158+
5159+-- | Whether to bypass the safety checks for the job\'s temporary directory.
5160+-- Use with caution.
5161+reBypassTempDirValidation :: Lens' RuntimeEnvironment (Maybe Bool)
5162+reBypassTempDirValidation
5163+ = lens _reBypassTempDirValidation
5164+ (\ s a -> s{_reBypassTempDirValidation = a})
5165+
5166+-- | The email address of the service account to run the job as.
5167+reServiceAccountEmail :: Lens' RuntimeEnvironment (Maybe Text)
5168+reServiceAccountEmail
5169+ = lens _reServiceAccountEmail
5170+ (\ s a -> s{_reServiceAccountEmail = a})
5171+
5172+-- | The maximum number of Google Compute Engine instances to be made
5173+-- available to your pipeline during execution, from 1 to 1000.
5174+reMaxWorkers :: Lens' RuntimeEnvironment (Maybe Int32)
5175+reMaxWorkers
5176+ = lens _reMaxWorkers (\ s a -> s{_reMaxWorkers = a})
5177+ . mapping _Coerce
5178+
5179+-- | The Cloud Storage path to use for temporary files. Must be a valid Cloud
5180+-- Storage URL, beginning with \`gs:\/\/\`.
5181+reTempLocation :: Lens' RuntimeEnvironment (Maybe Text)
5182+reTempLocation
5183+ = lens _reTempLocation
5184+ (\ s a -> s{_reTempLocation = a})
5185+
5186+instance FromJSON RuntimeEnvironment where
5187+ parseJSON
5188+ = withObject "RuntimeEnvironment"
5189+ (\ o ->
5190+ RuntimeEnvironment' <$>
5191+ (o .:? "zone") <*> (o .:? "bypassTempDirValidation")
5192+ <*> (o .:? "serviceAccountEmail")
5193+ <*> (o .:? "maxWorkers")
5194+ <*> (o .:? "tempLocation"))
5195+
5196+instance ToJSON RuntimeEnvironment where
5197+ toJSON RuntimeEnvironment'{..}
5198+ = object
5199+ (catMaybes
5200+ [("zone" .=) <$> _reZone,
5201+ ("bypassTempDirValidation" .=) <$>
5202+ _reBypassTempDirValidation,
5203+ ("serviceAccountEmail" .=) <$>
5204+ _reServiceAccountEmail,
5205+ ("maxWorkers" .=) <$> _reMaxWorkers,
5206+ ("tempLocation" .=) <$> _reTempLocation])
5207+
49355208-- | CounterMetadata includes all static non-name non-value counter
49365209-- attributes.
49375210--
@@ -5479,11 +5752,12 @@ instance ToJSON
54795752 PartialGroupByKeyInstructionValueCombiningFn where
54805753 toJSON = toJSON . _pgbkivcfAddtional
54815754
5482−-- | Defines a job to be run by the Dataflow service.
5755+-- | Defines a job to be run by the Cloud Dataflow service.
54835756--
54845757-- /See:/ 'job' smart constructor.
54855758data Job = Job'
54865759 { _jRequestedState :: !(Maybe Text)
5760+ , _jLocation :: !(Maybe Text)
54875761 , _jEnvironment :: !(Maybe Environment)
54885762 , _jClientRequestId :: !(Maybe Text)
54895763 , _jCurrentState :: !(Maybe Text)
@@ -5508,6 +5782,8 @@ data Job = Job'
55085782--
55095783-- * 'jRequestedState'
55105784--
5785+-- * 'jLocation'
5786+--
55115787-- * 'jEnvironment'
55125788--
55135789-- * 'jClientRequestId'
@@ -5544,6 +5820,7 @@ job
55445820job =
55455821 Job'
55465822 { _jRequestedState = Nothing
5823+ , _jLocation = Nothing
55475824 , _jEnvironment = Nothing
55485825 , _jClientRequestId = Nothing
55495826 , _jCurrentState = Nothing
@@ -5562,45 +5839,52 @@ job =
55625839 , _jCreateTime = Nothing
55635840 }
55645841
5565−-- | The job\'s requested state. UpdateJob may be used to switch between the
5566−-- JOB_STATE_STOPPED and JOB_STATE_RUNNING states, by setting
5567−-- requested_state. UpdateJob may also be used to directly set a job\'s
5568−-- requested state to JOB_STATE_CANCELLED or JOB_STATE_DONE, irrevocably
5569−-- terminating the job if it has not already reached a terminal state.
5842+-- | The job\'s requested state. \`UpdateJob\` may be used to switch between
5843+-- the \`JOB_STATE_STOPPED\` and \`JOB_STATE_RUNNING\` states, by setting
5844+-- requested_state. \`UpdateJob\` may also be used to directly set a job\'s
5845+-- requested state to \`JOB_STATE_CANCELLED\` or \`JOB_STATE_DONE\`,
5846+-- irrevocably terminating the job if it has not already reached a terminal
5847+-- state.
55705848jRequestedState :: Lens' Job (Maybe Text)
55715849jRequestedState
55725850 = lens _jRequestedState
55735851 (\ s a -> s{_jRequestedState = a})
55745852
5575−-- | Environment for the job.
5853+-- | The location that contains this job.
5854+jLocation :: Lens' Job (Maybe Text)
5855+jLocation
5856+ = lens _jLocation (\ s a -> s{_jLocation = a})
5857+
5858+-- | The environment for the job.
55765859jEnvironment :: Lens' Job (Maybe Environment)
55775860jEnvironment
55785861 = lens _jEnvironment (\ s a -> s{_jEnvironment = a})
55795862
5580−-- | Client\'s unique identifier of the job, re-used by SDK across retried
5863+-- | The client\'s unique identifier of the job, re-used across retried
55815864-- attempts. If this field is set, the service will ensure its uniqueness.
5582−-- That is, the request to create a job will fail if the service has
5583−-- knowledge of a previously submitted job with the same client\'s id and
5584−-- job name. The caller may, for example, use this field to ensure
5585−-- idempotence of job creation across retried attempts to create a job. By
5586−-- default, the field is empty and, in that case, the service ignores it.
5865+-- The request to create a job will fail if the service has knowledge of a
5866+-- previously submitted job with the same client\'s ID and job name. The
5867+-- caller may use this field to ensure idempotence of job creation across
5868+-- retried attempts to create a job. By default, the field is empty and, in
5869+-- that case, the service ignores it.
55875870jClientRequestId :: Lens' Job (Maybe Text)
55885871jClientRequestId
55895872 = lens _jClientRequestId
55905873 (\ s a -> s{_jClientRequestId = a})
55915874
5592−-- | The current state of the job. Jobs are created in the JOB_STATE_STOPPED
5593−-- state unless otherwise specified. A job in the JOB_STATE_RUNNING state
5594−-- may asynchronously enter a terminal state. Once a job has reached a
5595−-- terminal state, no further state updates may be made. This field may be
5596−-- mutated by the Dataflow service; callers cannot mutate it.
5875+-- | The current state of the job. Jobs are created in the
5876+-- \`JOB_STATE_STOPPED\` state unless otherwise specified. A job in the
5877+-- \`JOB_STATE_RUNNING\` state may asynchronously enter a terminal state.
5878+-- After a job has reached a terminal state, no further state updates may
5879+-- be made. This field may be mutated by the Cloud Dataflow service;
5880+-- callers cannot mutate it.
55975881jCurrentState :: Lens' Job (Maybe Text)
55985882jCurrentState
55995883 = lens _jCurrentState
56005884 (\ s a -> s{_jCurrentState = a})
56015885
56025886-- | If another job is an update of this job (and thus, this job is in
5603−-- JOB_STATE_UPDATED), this field will contain the ID of that job.
5887+-- \`JOB_STATE_UPDATED\`), this field contains the ID of that job.
56045888jReplacedByJobId :: Lens' Job (Maybe Text)
56055889jReplacedByJobId
56065890 = lens _jReplacedByJobId
@@ -5624,29 +5908,30 @@ jSteps
56245908 = lens _jSteps (\ s a -> s{_jSteps = a}) . _Default .
56255909 _Coerce
56265910
5627−-- | Information about how the Dataflow service will actually run the job.
5911+-- | Information about how the Cloud Dataflow service will run the job.
56285912jExecutionInfo :: Lens' Job (Maybe JobExecutionInfo)
56295913jExecutionInfo
56305914 = lens _jExecutionInfo
56315915 (\ s a -> s{_jExecutionInfo = a})
56325916
5633−-- | The user-specified Dataflow job name. Only one Job with a given name may
5634−-- exist in a project at any given time. If a caller attempts to create a
5635−-- Job with the same name as an already-existing Job, the attempt will
5636−-- return the existing Job. The name must match the regular expression
5637−-- [a-z]([-a-z0-9]{0,38}[a-z0-9])?
5917+-- | The user-specified Cloud Dataflow job name. Only one Job with a given
5918+-- name may exist in a project at any given time. If a caller attempts to
5919+-- create a Job with the same name as an already-existing Job, the attempt
5920+-- returns the existing Job. The name must match the regular expression
5921+-- \`[a-z]([-a-z0-9]{0,38}[a-z0-9])?\`
56385922jName :: Lens' Job (Maybe Text)
56395923jName = lens _jName (\ s a -> s{_jName = a})
56405924
5641−-- | Map of transform name prefixes of the job to be replaced to the
5925+-- | The map of transform name prefixes of the job to be replaced to the
56425926-- corresponding name prefixes of the new job.
56435927jTransformNameMApping :: Lens' Job (Maybe JobTransformNameMApping)
56445928jTransformNameMApping
56455929 = lens _jTransformNameMApping
56465930 (\ s a -> s{_jTransformNameMApping = a})
56475931
5648−-- | The unique ID of this job. This field is set by the Dataflow service
5649−-- when the Job is created, and is immutable for the life of the Job.
5932+-- | The unique ID of this job. This field is set by the Cloud Dataflow
5933+-- service when the Job is created, and is immutable for the life of the
5934+-- job.
56505935jId :: Lens' Job (Maybe Text)
56515936jId = lens _jId (\ s a -> s{_jId = a})
56525937
@@ -5659,12 +5944,12 @@ jId = lens _jId (\ s a -> s{_jId = a})
56595944jLabels :: Lens' Job (Maybe JobLabels)
56605945jLabels = lens _jLabels (\ s a -> s{_jLabels = a})
56615946
5662−-- | The project which owns the job.
5947+-- | The ID of the Cloud Platform project that the job belongs to.
56635948jProjectId :: Lens' Job (Maybe Text)
56645949jProjectId
56655950 = lens _jProjectId (\ s a -> s{_jProjectId = a})
56665951
5667−-- | The type of dataflow job.
5952+-- | The type of Cloud Dataflow job.
56685953jType :: Lens' Job (Maybe Text)
56695954jType = lens _jType (\ s a -> s{_jType = a})
56705955
@@ -5674,17 +5959,17 @@ jCurrentStateTime
56745959 = lens _jCurrentStateTime
56755960 (\ s a -> s{_jCurrentStateTime = a})
56765961
5677−-- | If this job is an update of an existing job, this field will be the ID
5678−-- of the job it replaced. When sending a CreateJobRequest, you can update
5679−-- a job by specifying it here. The job named here will be stopped, and its
5680−-- intermediate state transferred to this job.
5962+-- | If this job is an update of an existing job, this field is the job ID of
5963+-- the job it replaced. When sending a \`CreateJobRequest\`, you can update
5964+-- a job by specifying it here. The job named here is stopped, and its
5965+-- intermediate state is transferred to this job.
56815966jReplaceJobId :: Lens' Job (Maybe Text)
56825967jReplaceJobId
56835968 = lens _jReplaceJobId
56845969 (\ s a -> s{_jReplaceJobId = a})
56855970
5686−-- | Timestamp when job was initially created. Immutable, set by the Dataflow
5687−-- service.
5971+-- | The timestamp when the job was initially created. Immutable and set by
5972+-- the Cloud Dataflow service.
56885973jCreateTime :: Lens' Job (Maybe Text)
56895974jCreateTime
56905975 = lens _jCreateTime (\ s a -> s{_jCreateTime = a})
@@ -5694,7 +5979,8 @@ instance FromJSON Job where
56945979 = withObject "Job"
56955980 (\ o ->
56965981 Job' <$>
5697− (o .:? "requestedState") <*> (o .:? "environment")
5982+ (o .:? "requestedState") <*> (o .:? "location") <*>
5983+ (o .:? "environment")
56985984 <*> (o .:? "clientRequestId")
56995985 <*> (o .:? "currentState")
57005986 <*> (o .:? "replacedByJobId")
@@ -5716,6 +6002,7 @@ instance ToJSON Job where
57166002 = object
57176003 (catMaybes
57186004 [("requestedState" .=) <$> _jRequestedState,
6005+ ("location" .=) <$> _jLocation,
57196006 ("environment" .=) <$> _jEnvironment,
57206007 ("clientRequestId" .=) <$> _jClientRequestId,
57216008 ("currentState" .=) <$> _jCurrentState,
@@ -5899,10 +6186,11 @@ instance FromJSON SinkSpec where
58996186instance ToJSON SinkSpec where
59006187 toJSON = toJSON . _sAddtional
59016188
5902−-- | Describes one particular pool of Dataflow workers to be instantiated by
5903−-- the Dataflow service in order to perform the computations required by a
5904−-- job. Note that a workflow job may use multiple pools, in order to match
5905−-- the various computational requirements of the various stages of the job.
6189+-- | Describes one particular pool of Cloud Dataflow workers to be
6190+-- instantiated by the Cloud Dataflow service in order to perform the
6191+-- computations required by a job. Note that a workflow job may use
6192+-- multiple pools, in order to match the various computational requirements
6193+-- of the various stages of the job.
59066194--
59076195-- /See:/ 'workerPool' smart constructor.
59086196data WorkerPool = WorkerPool'
@@ -6024,7 +6312,7 @@ wpDiskSizeGb
60246312 = lens _wpDiskSizeGb (\ s a -> s{_wpDiskSizeGb = a})
60256313 . mapping _Coerce
60266314
6027−-- | The kind of the worker pool; currently only \'harness\' and \'shuffle\'
6315+-- | The kind of the worker pool; currently only \`harness\` and \`shuffle\`
60286316-- are supported.
60296317wpKind :: Lens' WorkerPool (Maybe Text)
60306318wpKind = lens _wpKind (\ s a -> s{_wpKind = a})
@@ -6106,16 +6394,17 @@ wpDiskType
61066394 = lens _wpDiskType (\ s a -> s{_wpDiskType = a})
61076395
61086396-- | Sets the policy for determining when to turndown worker pool. Allowed
6109−-- values are: TEARDOWN_ALWAYS, TEARDOWN_ON_SUCCESS, and TEARDOWN_NEVER.
6110−-- TEARDOWN_ALWAYS means workers are always torn down regardless of whether
6111−-- the job succeeds. TEARDOWN_ON_SUCCESS means workers are torn down if the
6112−-- job succeeds. TEARDOWN_NEVER means the workers are never torn down. If
6113−-- the workers are not torn down by the service, they will continue to run
6114−-- and use Google Compute Engine VM resources in the user\'s project until
6115−-- they are explicitly terminated by the user. Because of this, Google
6116−-- recommends using the TEARDOWN_ALWAYS policy except for small, manually
6117−-- supervised test jobs. If unknown or unspecified, the service will
6118−-- attempt to choose a reasonable default.
6397+-- values are: \`TEARDOWN_ALWAYS\`, \`TEARDOWN_ON_SUCCESS\`, and
6398+-- \`TEARDOWN_NEVER\`. \`TEARDOWN_ALWAYS\` means workers are always torn
6399+-- down regardless of whether the job succeeds. \`TEARDOWN_ON_SUCCESS\`
6400+-- means workers are torn down if the job succeeds. \`TEARDOWN_NEVER\`
6401+-- means the workers are never torn down. If the workers are not torn down
6402+-- by the service, they will continue to run and use Google Compute Engine
6403+-- VM resources in the user\'s project until they are explicitly terminated
6404+-- by the user. Because of this, Google recommends using the
6405+-- \`TEARDOWN_ALWAYS\` policy except for small, manually supervised test
6406+-- jobs. If unknown or unspecified, the service will attempt to choose a
6407+-- reasonable default.
61196408wpTeardownPolicy :: Lens' WorkerPool (Maybe Text)
61206409wpTeardownPolicy
61216410 = lens _wpTeardownPolicy
@@ -6134,8 +6423,8 @@ wpPoolArgs :: Lens' WorkerPool (Maybe WorkerPoolPoolArgs)
61346423wpPoolArgs
61356424 = lens _wpPoolArgs (\ s a -> s{_wpPoolArgs = a})
61366425
6137−-- | Docker container image that executes Dataflow worker harness, residing
6138−-- in Google Container Registry. Required.
6426+-- | Required. Docker container image that executes the Cloud Dataflow worker
6427+-- harness, residing in Google Container Registry.
61396428wpWorkerHarnessContainerImage :: Lens' WorkerPool (Maybe Text)
61406429wpWorkerHarnessContainerImage
61416430 = lens _wpWorkerHarnessContainerImage
@@ -6204,7 +6493,7 @@ instance ToJSON WorkerPool where
62046493 _wpWorkerHarnessContainerImage,
62056494 ("dataDisks" .=) <$> _wpDataDisks])
62066495
6207−-- | Defines a particular step within a Dataflow job. A job consists of
6496+-- | Defines a particular step within a Cloud Dataflow job. A job consists of
62086497-- multiple steps, each of which performs some specific operation as part
62096498-- of the overall job. Data is typically passed from one step to another as
62106499-- part of the job. Here\'s an example of a sequence of steps which
@@ -6214,8 +6503,8 @@ instance ToJSON WorkerPool where
62146503-- value and extract an element-specific key value. * Group elements with
62156504-- the same key into a single element with that key, transforming a
62166505-- multiply-keyed collection into a uniquely-keyed collection. * Write the
6217−-- elements out to some data sink. (Note that the Dataflow service may be
6218−-- used to run many different types of jobs, not just Map-Reduce).
6506+-- elements out to some data sink. Note that the Cloud Dataflow service may
6507+-- be used to run many different types of jobs, not just Map-Reduce.
62196508--
62206509-- /See:/ 'step' smart constructor.
62216510data Step = Step'
@@ -6242,12 +6531,12 @@ step =
62426531 , _sProperties = Nothing
62436532 }
62446533
6245−-- | The kind of step in the dataflow Job.
6534+-- | The kind of step in the Cloud Dataflow job.
62466535sKind :: Lens' Step (Maybe Text)
62476536sKind = lens _sKind (\ s a -> s{_sKind = a})
62486537
6249−-- | Name identifying the step. This must be unique for each step with
6250−-- respect to all other steps in the dataflow Job.
6538+-- | The name that identifies the step. This must be unique for each step
6539+-- with respect to all other steps in the Cloud Dataflow job.
62516540sName :: Lens' Step (Maybe Text)
62526541sName = lens _sName (\ s a -> s{_sName = a})
62536542
@@ -6272,12 +6561,13 @@ instance ToJSON Step where
62726561 [("kind" .=) <$> _sKind, ("name" .=) <$> _sName,
62736562 ("properties" .=) <$> _sProperties])
62746563
6275−-- | Packages that need to be installed in order for a worker to run the
6276−-- steps of the Dataflow job which will be assigned to its worker pool.
6277−-- This is the mechanism by which the SDK causes code to be loaded onto the
6278−-- workers. For example, the Dataflow Java SDK might use this to install
6279−-- jars containing the user\'s code and all of the various dependencies
6280−-- (libraries, data files, etc) required in order for that code to run.
6564+-- | The packages that must be installed in order for a worker to run the
6565+-- steps of the Cloud Dataflow job that will be assigned to its worker
6566+-- pool. This is the mechanism by which the Cloud Dataflow SDK causes code
6567+-- to be loaded onto the workers. For example, the Cloud Dataflow Java SDK
6568+-- might use this to install jars containing the user\'s code and all of
6569+-- the various dependencies (libraries, data files, etc.) required in order
6570+-- for that code to run.
62816571--
62826572-- /See:/ 'package' smart constructor.
62836573data Package = Package'
@@ -6538,6 +6828,29 @@ instance ToJSON SourceFork where
65386828 ("primary" .=) <$> _sfPrimary,
65396829 ("residualSource" .=) <$> _sfResidualSource])
65406830
6831+-- | Service-side response to WorkerMessage reporting resource utilization.
6832+--
6833+-- /See:/ 'resourceUtilizationReportResponse' smart constructor.
6834+data ResourceUtilizationReportResponse =
6835+ ResourceUtilizationReportResponse'
6836+ deriving (Eq,Show,Data,Typeable,Generic)
6837+
6838+-- | Creates a value of 'ResourceUtilizationReportResponse' with the minimum fields required to make a request.
6839+--
6840+resourceUtilizationReportResponse
6841+ :: ResourceUtilizationReportResponse
6842+resourceUtilizationReportResponse = ResourceUtilizationReportResponse'
6843+
6844+instance FromJSON ResourceUtilizationReportResponse
6845+ where
6846+ parseJSON
6847+ = withObject "ResourceUtilizationReportResponse"
6848+ (\ o -> pure ResourceUtilizationReportResponse')
6849+
6850+instance ToJSON ResourceUtilizationReportResponse
6851+ where
6852+ toJSON = const emptyObject
6853+
65416854-- | Describes full or partial data disk assignment information of the
65426855-- computation ranges.
65436856--
@@ -6591,13 +6904,14 @@ instance ToJSON StreamingComputationRanges where
65916904 [("rangeAssignments" .=) <$> _scrRangeAssignments,
65926905 ("computationId" .=) <$> _scrComputationId])
65936906
6594−-- | Response to a request to list Dataflow jobs. This may be a partial
6907+-- | Response to a request to list Cloud Dataflow jobs. This may be a partial
65956908-- response, depending on the page size in the ListJobsRequest.
65966909--
65976910-- /See:/ 'listJobsResponse' smart constructor.
65986911data ListJobsResponse = ListJobsResponse'
6599− { _ljrNextPageToken :: !(Maybe Text)
6600− , _ljrJobs :: !(Maybe [Job])
6912+ { _ljrNextPageToken :: !(Maybe Text)
6913+ , _ljrFailedLocation :: !(Maybe [FailedLocation])
6914+ , _ljrJobs :: !(Maybe [Job])
66016915 } deriving (Eq,Show,Data,Typeable,Generic)
66026916
66036917-- | Creates a value of 'ListJobsResponse' with the minimum fields required to make a request.
@@ -6606,12 +6920,15 @@ data ListJobsResponse = ListJobsResponse'
66066920--
66076921-- * 'ljrNextPageToken'
66086922--
6923+-- * 'ljrFailedLocation'
6924+--
66096925-- * 'ljrJobs'
66106926listJobsResponse
66116927 :: ListJobsResponse
66126928listJobsResponse =
66136929 ListJobsResponse'
66146930 { _ljrNextPageToken = Nothing
6931+ , _ljrFailedLocation = Nothing
66156932 , _ljrJobs = Nothing
66166933 }
66176934
@@ -6621,6 +6938,14 @@ ljrNextPageToken
66216938 = lens _ljrNextPageToken
66226939 (\ s a -> s{_ljrNextPageToken = a})
66236940
6941+-- | Zero or more messages describing locations that failed to respond.
6942+ljrFailedLocation :: Lens' ListJobsResponse [FailedLocation]
6943+ljrFailedLocation
6944+ = lens _ljrFailedLocation
6945+ (\ s a -> s{_ljrFailedLocation = a})
6946+ . _Default
6947+ . _Coerce
6948+
66246949-- | A subset of the requested job information.
66256950ljrJobs :: Lens' ListJobsResponse [Job]
66266951ljrJobs
@@ -6633,13 +6958,15 @@ instance FromJSON ListJobsResponse where
66336958 (\ o ->
66346959 ListJobsResponse' <$>
66356960 (o .:? "nextPageToken") <*>
6636− (o .:? "jobs" .!= mempty))
6961+ (o .:? "failedLocation" .!= mempty)
6962+ <*> (o .:? "jobs" .!= mempty))
66376963
66386964instance ToJSON ListJobsResponse where
66396965 toJSON ListJobsResponse'{..}
66406966 = object
66416967 (catMaybes
66426968 [("nextPageToken" .=) <$> _ljrNextPageToken,
6969+ ("failedLocation" .=) <$> _ljrFailedLocation,
66436970 ("jobs" .=) <$> _ljrJobs])
66446971
66456972-- | A source that records can be read and decoded from.
@@ -6815,6 +7142,7 @@ data WorkerMessage = WorkerMessage'
68157142 { _wmWorkerHealthReport :: !(Maybe WorkerHealthReport)
68167143 , _wmTime :: !(Maybe Text)
68177144 , _wmWorkerMessageCode :: !(Maybe WorkerMessageCode)
7145+ , _wmWorkerMetrics :: !(Maybe ResourceUtilizationReport)
68187146 , _wmLabels :: !(Maybe WorkerMessageLabels)
68197147 } deriving (Eq,Show,Data,Typeable,Generic)
68207148
@@ -6828,6 +7156,8 @@ data WorkerMessage = WorkerMessage'
68287156--
68297157-- * 'wmWorkerMessageCode'
68307158--
7159+-- * 'wmWorkerMetrics'
7160+--
68317161-- * 'wmLabels'
68327162workerMessage
68337163 :: WorkerMessage
@@ -6836,6 +7166,7 @@ workerMessage =
68367166 { _wmWorkerHealthReport = Nothing
68377167 , _wmTime = Nothing
68387168 , _wmWorkerMessageCode = Nothing
7169+ , _wmWorkerMetrics = Nothing
68397170 , _wmLabels = Nothing
68407171 }
68417172
@@ -6855,6 +7186,12 @@ wmWorkerMessageCode
68557186 = lens _wmWorkerMessageCode
68567187 (\ s a -> s{_wmWorkerMessageCode = a})
68577188
7189+-- | Resource metrics reported by workers.
7190+wmWorkerMetrics :: Lens' WorkerMessage (Maybe ResourceUtilizationReport)
7191+wmWorkerMetrics
7192+ = lens _wmWorkerMetrics
7193+ (\ s a -> s{_wmWorkerMetrics = a})
7194+
68587195-- | Labels are used to group WorkerMessages. For example, a worker_message
68597196-- about a particular container might have the labels: { \"JOB_ID\":
68607197-- \"2015-04-22\", \"WORKER_ID\": \"wordcount-vm-2015…\"
@@ -6872,6 +7209,7 @@ instance FromJSON WorkerMessage where
68727209 WorkerMessage' <$>
68737210 (o .:? "workerHealthReport") <*> (o .:? "time") <*>
68747211 (o .:? "workerMessageCode")
7212+ <*> (o .:? "workerMetrics")
68757213 <*> (o .:? "labels"))
68767214
68777215instance ToJSON WorkerMessage where
@@ -6881,6 +7219,7 @@ instance ToJSON WorkerMessage where
68817219 [("workerHealthReport" .=) <$> _wmWorkerHealthReport,
68827220 ("time" .=) <$> _wmTime,
68837221 ("workerMessageCode" .=) <$> _wmWorkerMessageCode,
7222+ ("workerMetrics" .=) <$> _wmWorkerMetrics,
68847223 ("labels" .=) <$> _wmLabels])
68857224
68867225-- | Location information for a specific key-range of a sharded computation.
@@ -7233,6 +7572,83 @@ instance ToJSON CustomSourceLocation where
72337572 = object
72347573 (catMaybes [("stateful" .=) <$> _cslStateful])
72357574
7575+-- | A metric value representing a distribution.
7576+--
7577+-- /See:/ 'distributionUpdate' smart constructor.
7578+data DistributionUpdate = DistributionUpdate'
7579+ { _duMax :: !(Maybe SplitInt64)
7580+ , _duCount :: !(Maybe SplitInt64)
7581+ , _duMin :: !(Maybe SplitInt64)
7582+ , _duSumOfSquares :: !(Maybe (Textual Double))
7583+ , _duSum :: !(Maybe SplitInt64)
7584+ } deriving (Eq,Show,Data,Typeable,Generic)
7585+
7586+-- | Creates a value of 'DistributionUpdate' with the minimum fields required to make a request.
7587+--
7588+-- Use one of the following lenses to modify other fields as desired:
7589+--
7590+-- * 'duMax'
7591+--
7592+-- * 'duCount'
7593+--
7594+-- * 'duMin'
7595+--
7596+-- * 'duSumOfSquares'
7597+--
7598+-- * 'duSum'
7599+distributionUpdate
7600+ :: DistributionUpdate
7601+distributionUpdate =
7602+ DistributionUpdate'
7603+ { _duMax = Nothing
7604+ , _duCount = Nothing
7605+ , _duMin = Nothing
7606+ , _duSumOfSquares = Nothing
7607+ , _duSum = Nothing
7608+ }
7609+
7610+-- | The maximum value present in the distribution.
7611+duMax :: Lens' DistributionUpdate (Maybe SplitInt64)
7612+duMax = lens _duMax (\ s a -> s{_duMax = a})
7613+
7614+-- | The count of the number of elements present in the distribution.
7615+duCount :: Lens' DistributionUpdate (Maybe SplitInt64)
7616+duCount = lens _duCount (\ s a -> s{_duCount = a})
7617+
7618+-- | The minimum value present in the distribution.
7619+duMin :: Lens' DistributionUpdate (Maybe SplitInt64)
7620+duMin = lens _duMin (\ s a -> s{_duMin = a})
7621+
7622+-- | Use a double since the sum of squares is likely to overflow int64.
7623+duSumOfSquares :: Lens' DistributionUpdate (Maybe Double)
7624+duSumOfSquares
7625+ = lens _duSumOfSquares
7626+ (\ s a -> s{_duSumOfSquares = a})
7627+ . mapping _Coerce
7628+
7629+-- | Use an int64 since we\'d prefer the added precision. If overflow is a
7630+-- common problem we can detect it and use an additional int64 or a double.
7631+duSum :: Lens' DistributionUpdate (Maybe SplitInt64)
7632+duSum = lens _duSum (\ s a -> s{_duSum = a})
7633+
7634+instance FromJSON DistributionUpdate where
7635+ parseJSON
7636+ = withObject "DistributionUpdate"
7637+ (\ o ->
7638+ DistributionUpdate' <$>
7639+ (o .:? "max") <*> (o .:? "count") <*> (o .:? "min")
7640+ <*> (o .:? "sumOfSquares")
7641+ <*> (o .:? "sum"))
7642+
7643+instance ToJSON DistributionUpdate where
7644+ toJSON DistributionUpdate'{..}
7645+ = object
7646+ (catMaybes
7647+ [("max" .=) <$> _duMax, ("count" .=) <$> _duCount,
7648+ ("min" .=) <$> _duMin,
7649+ ("sumOfSquares" .=) <$> _duSumOfSquares,
7650+ ("sum" .=) <$> _duSum])
7651+
72367652--
72377653-- /See:/ 'sourceBaseSpecsItem' smart constructor.
72387654newtype SourceBaseSpecsItem = SourceBaseSpecsItem'
@@ -7681,8 +8097,8 @@ instance ToJSON SideInputInfo where
76818097 [("tag" .=) <$> _siiTag, ("kind" .=) <$> _siiKind,
76828098 ("sources" .=) <$> _siiSources])
76838099
7684−-- | Additional information about how a Dataflow job will be executed which
7685−-- isn’t contained in the submitted job.
8100+-- | Additional information about how a Cloud Dataflow job will be executed
8101+-- that isn\'t contained in the submitted job.
76868102--
76878103-- /See:/ 'jobExecutionInfo' smart constructor.
76888104newtype JobExecutionInfo = JobExecutionInfo'
@@ -7979,6 +8395,7 @@ data PubsubLocation = PubsubLocation'
79798395 , _plTimestampLabel :: !(Maybe Text)
79808396 , _plIdLabel :: !(Maybe Text)
79818397 , _plTopic :: !(Maybe Text)
8398+ , _plWithAttributes :: !(Maybe Bool)
79828399 , _plSubscription :: !(Maybe Text)
79838400 } deriving (Eq,Show,Data,Typeable,Generic)
79848401
@@ -7996,6 +8413,8 @@ data PubsubLocation = PubsubLocation'
79968413--
79978414-- * 'plTopic'
79988415--
8416+-- * 'plWithAttributes'
8417+--
79998418-- * 'plSubscription'
80008419pubsubLocation
80018420 :: PubsubLocation
@@ -8006,6 +8425,7 @@ pubsubLocation =
80068425 , _plTimestampLabel = Nothing
80078426 , _plIdLabel = Nothing
80088427 , _plTopic = Nothing
8428+ , _plWithAttributes = Nothing
80098429 , _plSubscription = Nothing
80108430 }
80118431
@@ -8039,6 +8459,12 @@ plIdLabel
80398459plTopic :: Lens' PubsubLocation (Maybe Text)
80408460plTopic = lens _plTopic (\ s a -> s{_plTopic = a})
80418461
8462+-- | If true, then the client has requested to get pubsub attributes.
8463+plWithAttributes :: Lens' PubsubLocation (Maybe Bool)
8464+plWithAttributes
8465+ = lens _plWithAttributes
8466+ (\ s a -> s{_plWithAttributes = a})
8467+
80428468-- | A pubsub subscription, in the form of
80438469-- \"pubsub.googleapis.com\/subscriptions\/ \/\"
80448470plSubscription :: Lens' PubsubLocation (Maybe Text)
@@ -8056,6 +8482,7 @@ instance FromJSON PubsubLocation where
80568482 <*> (o .:? "timestampLabel")
80578483 <*> (o .:? "idLabel")
80588484 <*> (o .:? "topic")
8485+ <*> (o .:? "withAttributes")
80598486 <*> (o .:? "subscription"))
80608487
80618488instance ToJSON PubsubLocation where
@@ -8068,6 +8495,7 @@ instance ToJSON PubsubLocation where
80688495 ("timestampLabel" .=) <$> _plTimestampLabel,
80698496 ("idLabel" .=) <$> _plIdLabel,
80708497 ("topic" .=) <$> _plTopic,
8498+ ("withAttributes" .=) <$> _plWithAttributes,
80718499 ("subscription" .=) <$> _plSubscription])
80728500
80738501-- | A metric value representing a list of floating point numbers.