gcp-workflow-example.yaml

main:
  params: [args]
  steps:
    - read_runtime_args:
        assign:
          - project: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
          - location: ${sys.get_env("GOOGLE_CLOUD_LOCATION")}
          - repository: "custom-jobs"
          - depsImage: "image-caption-gpu-deps"
          - flexImage: "image-caption-gpu-flex"
          - jobId: "workflow-dataflow-flex-run"
    - run_flex_dataflow_job:
        call: googleapis.dataflow.v1b3.projects.locations.flexTemplates.launch
        # If you use `projects.locations.flexTemplates.launch`, you can directly pass the json template to `containerSpecGcsPath`
        # and waive all the parameters and runtime settings.
        # I didn't know this^ convenient API when i was writing this GCP workflow.
        result: launchResult
        args:
          location: ${location}
          projectId: ${project}
          body:
            # The following parameters are ordered to align with Google's Official API.
            # https://github.com/googleapis/googleapis/blob/1cfcea4fbef317c44cc13d73017a0b0462c4737d/google/dataflow/v1beta3/templates.proto#L118
            launchParameter:
              jobName: ${jobId + "-" + string(uuid.generate())}
              containerSpec:
                image: ${ "us-central1-docker.pkg.dev/" + project + "/custom-jobs/" + flexImage }
                sdkInfo:
                  language: PYTHON
              parameters:
                sdk_location: container
                sdk_container_image: ${ "us-central1-docker.pkg.dev/" + project + "/custom-jobs/" + depsImage" } # Put the image where your code logic is saved in.
                runner: DataflowRunner
                job_name_prefix: "image-caption"
                machine_type: n1-highmem-8
                experiment: use_runner_v2
                disk_size_gb: '50'
                ## Add your job-specific params below
                dry_run: ${args.dry_run}
              environment:
                numWorkers: 1
                maxWorkers: 10
                tempLocation: ${ "gs://" + project + "dataflow-temp/image-caption" }
                stagingLocation: ${ "gs://" + project + "dataflow-staging/image-caption" }