From ba59579141916d25a9d371ca945bc5e28289ce4e Mon Sep 17 00:00:00 2001 From: averikitsch Date: Mon, 15 Apr 2019 18:49:07 -0700 Subject: [PATCH 1/7] Add task with authentication sample --- appengine/flexible/tasks/Dockerfile | 17 ++++ .../tasks/create_http_task_with_token.py | 80 +++++++++++++++++++ .../tasks/create_http_task_with_token_test.py | 28 +++++++ appengine/flexible/tasks/requirements.txt | 2 +- 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 appengine/flexible/tasks/Dockerfile create mode 100644 appengine/flexible/tasks/create_http_task_with_token.py create mode 100644 appengine/flexible/tasks/create_http_task_with_token_test.py diff --git a/appengine/flexible/tasks/Dockerfile b/appengine/flexible/tasks/Dockerfile new file mode 100644 index 0000000000..64f160d825 --- /dev/null +++ b/appengine/flexible/tasks/Dockerfile @@ -0,0 +1,17 @@ +# Use the official Python image. +# https://hub.docker.com/_/python +FROM python:3.7 + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . . + +# Install production dependencies. +RUN pip install Flask gunicorn + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 main:app diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py new file mode 100644 index 0000000000..42188b550a --- /dev/null +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -0,0 +1,80 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import datetime + + +def create_http_task(project, + queue, + location, + url, + payload=None, + in_seconds=None): + # [START cloud_tasks_create_http_task_with_token] + """Create a task for a given queue with an arbitrary payload.""" + + from google.cloud import tasks_v2beta3 + from google.protobuf import timestamp_pb2 + + # Create a client. + client = tasks_v2beta3.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # queue = 'my-appengine-queue' + # location = 'us-central1' + # url = 'https://.appspot.com/example_task_handler' + # payload = 'hello' + + # Construct the fully qualified queue name. + parent = client.queue_path(project, location, queue) + + # Construct the request body. + task = { + 'http_request': { # Specify the type of request. + 'http_method': 'POST', + 'url': url, # The full url path that the task will be sent to. + 'oidc_token': { + 'service_account_email': + 'client_id@project_id.iam.gserviceaccount.com' + } + } + } + if payload is not None: + # The API expects a payload of type bytes. + converted_payload = payload.encode() + + # Add the payload to the request. + task['http_request']['body'] = converted_payload + + if in_seconds is not None: + # Convert "seconds from now" into an rfc3339 datetime string. + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) + + # Create Timestamp protobuf. + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(d) + + # Add the timestamp to the tasks. + task['schedule_time'] = timestamp + + # Use the client to build and send the task. + response = client.create_task(parent, task) + + print('Created task {}'.format(response.name)) + return response +# [END cloud_tasks_create_http_task_with_token] diff --git a/appengine/flexible/tasks/create_http_task_with_token_test.py b/appengine/flexible/tasks/create_http_task_with_token_test.py new file mode 100644 index 0000000000..4d51fd0468 --- /dev/null +++ b/appengine/flexible/tasks/create_http_task_with_token_test.py @@ -0,0 +1,28 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import create_http_task_with_token + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') + + +def test_create_http_task_with_token(): + url = 'https://example.com/example_task_handler' + result = create_http_task_with_token.create_http_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) + assert TEST_QUEUE_NAME in result.name diff --git a/appengine/flexible/tasks/requirements.txt b/appengine/flexible/tasks/requirements.txt index 9aadd8a774..fe50a5aa3b 100644 --- a/appengine/flexible/tasks/requirements.txt +++ b/appengine/flexible/tasks/requirements.txt @@ -1,3 +1,3 @@ Flask==1.0.2 gunicorn==19.9.0 -google-cloud-tasks==0.6.0 +google-cloud-tasks==0.7.0 From a832385646d5921c45303d88cb358ba7c0318d9e Mon Sep 17 00:00:00 2001 From: averikitsch Date: Mon, 15 Apr 2019 18:59:45 -0700 Subject: [PATCH 2/7] Fix linting --- appengine/flexible/tasks/create_http_task_with_token.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 42188b550a..5642233e62 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -14,10 +14,8 @@ from __future__ import print_function -import argparse import datetime - def create_http_task(project, queue, location, From a5554ef495c7474114be9324bbdef64d79a1354c Mon Sep 17 00:00:00 2001 From: averikitsch Date: Mon, 15 Apr 2019 18:59:45 -0700 Subject: [PATCH 3/7] Fix linting --- appengine/flexible/tasks/create_http_task_with_token.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 42188b550a..5642233e62 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -14,10 +14,8 @@ from __future__ import print_function -import argparse import datetime - def create_http_task(project, queue, location, From 4c5d6b4e07b95158ee67c0cfaba9b3d2853d6f22 Mon Sep 17 00:00:00 2001 From: averikitsch Date: Tue, 16 Apr 2019 09:14:16 -0700 Subject: [PATCH 4/7] Fix spacing --- appengine/flexible/tasks/create_http_task_with_token.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 5642233e62..2c4f851e02 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -16,6 +16,7 @@ import datetime + def create_http_task(project, queue, location, @@ -52,6 +53,7 @@ def create_http_task(project, } } } + if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() From 0e69eef12dd89e307983016b7db38d9887a97fe7 Mon Sep 17 00:00:00 2001 From: averikitsch Date: Tue, 16 Apr 2019 09:35:32 -0700 Subject: [PATCH 5/7] Update tests with service account --- .kokoro/system_tests.sh | 1 + appengine/flexible/tasks/create_http_task_with_token.py | 6 +++--- .../flexible/tasks/create_http_task_with_token_test.py | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh index 80b9a3173e..aae4cda670 100755 --- a/.kokoro/system_tests.sh +++ b/.kokoro/system_tests.sh @@ -25,6 +25,7 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") source ./testing/test-env.sh export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json +source ${KOKORO_GFILE_DIR}/tasks-service-account.sh # Run Cloud SQL proxy, if required if [ -n "${CLOUD_SQL_PROXY}" ]; then diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/appengine/flexible/tasks/create_http_task_with_token.py index 2c4f851e02..2849492895 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/appengine/flexible/tasks/create_http_task_with_token.py @@ -21,6 +21,7 @@ def create_http_task(project, queue, location, url, + service_account_email, payload=None, in_seconds=None): # [START cloud_tasks_create_http_task_with_token] @@ -48,12 +49,11 @@ def create_http_task(project, 'http_method': 'POST', 'url': url, # The full url path that the task will be sent to. 'oidc_token': { - 'service_account_email': - 'client_id@project_id.iam.gserviceaccount.com' + 'service_account_email': service_account_email } } } - + if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() diff --git a/appengine/flexible/tasks/create_http_task_with_token_test.py b/appengine/flexible/tasks/create_http_task_with_token_test.py index 4d51fd0468..aa33b672ad 100644 --- a/appengine/flexible/tasks/create_http_task_with_token_test.py +++ b/appengine/flexible/tasks/create_http_task_with_token_test.py @@ -19,10 +19,14 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') +TEST_SERVICE_ACCOUNT = os.getenv('CLOUD_RUN_INVOKER_SERVICE_ACCOUNT') def test_create_http_task_with_token(): url = 'https://example.com/example_task_handler' - result = create_http_task_with_token.create_http_task( - TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION, url) + result = create_http_task_with_token.create_http_task(TEST_PROJECT_ID, + TEST_QUEUE_NAME, + TEST_LOCATION, + url, + TEST_SERVICE_ACCOUNT) assert TEST_QUEUE_NAME in result.name From 3fb01f6cbff20e83c9ba2fefe2edc5c8c50224dc Mon Sep 17 00:00:00 2001 From: averikitsch Date: Thu, 18 Apr 2019 11:23:27 -0700 Subject: [PATCH 6/7] Move samples and update READMEs --- .kokoro/system_tests.sh | 1 - appengine/flexible/tasks/README.md | 21 ------------ tasks/README.md | 34 +++---------------- .../create_http_task_with_token.py | 2 +- .../create_http_task_with_token_test.py | 3 +- 5 files changed, 8 insertions(+), 53 deletions(-) rename {appengine/flexible/tasks => tasks}/create_http_task_with_token.py (97%) rename {appengine/flexible/tasks => tasks}/create_http_task_with_token_test.py (93%) diff --git a/.kokoro/system_tests.sh b/.kokoro/system_tests.sh index aae4cda670..80b9a3173e 100755 --- a/.kokoro/system_tests.sh +++ b/.kokoro/system_tests.sh @@ -25,7 +25,6 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt") source ./testing/test-env.sh export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json -source ${KOKORO_GFILE_DIR}/tasks-service-account.sh # Run Cloud SQL proxy, if required if [ -n "${CLOUD_SQL_PROXY}" ]; then diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md index 303fbfcfca..c2888ba647 100644 --- a/appengine/flexible/tasks/README.md +++ b/appengine/flexible/tasks/README.md @@ -101,24 +101,3 @@ endpoint, with a payload specified: ``` python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --payload=hello ``` - -### Using HTTP Push Queues - -Set an environment variable for the endpoint to your task handler. This is an -example url to send requests to the App Engine task handler: -``` -export URL=https://.appspot.com/example_task_handler -``` - -Running the sample will create a task and send the task to the specific URL -endpoint, with a payload specified: - -``` -python create_http_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --url=$URL --payload=hello -``` - -Now view that the payload was received and verify the payload: - -``` -gcloud app logs read -``` diff --git a/tasks/README.md b/tasks/README.md index 61ac422cb2..00503ccd39 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -12,8 +12,11 @@ App Engine queues push tasks to an App Engine HTTP target. This directory contains both the App Engine app to deploy, as well as the snippets to run locally to push tasks to it, which could also be called on App Engine. -`create_app_engine_queue_task.py` is a simple command-line program to create -tasks to be pushed to the App Engine app. +`create_http_task.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint. + +`create_http_task_with_token.py` is a simple command-line program to create +tasks to be pushed to an URL endpoint with authorization header. `main.py` is the main App Engine app. This app serves as an endpoint to receive App Engine task attempts. @@ -41,33 +44,6 @@ gcloud beta tasks queues create-app-engine-queue my-appengine-queue Note: A newly created queue will route to the default App Engine service and version unless configured to do otherwise. -## Deploying the App Engine App - -Deploy the App Engine app with gcloud: - -* To deploy to the Standard environment: - ``` - gcloud app deploy app.yaml - ``` -* To deploy to the Flexible environment: - ``` - gcloud app deploy app.flexible.yaml - ``` - -Verify the index page is serving: - -``` -gcloud app browse -``` - -The App Engine app serves as a target for the push requests. It has an -endpoint `/example_task_handler` that reads the payload (i.e., the request body) -of the HTTP POST request and logs it. The log output can be viewed with: - -``` -gcloud app logs read -``` - ## Run the Sample Using the Command Line Set environment variables: diff --git a/appengine/flexible/tasks/create_http_task_with_token.py b/tasks/create_http_task_with_token.py similarity index 97% rename from appengine/flexible/tasks/create_http_task_with_token.py rename to tasks/create_http_task_with_token.py index 2849492895..1b79a9b3fc 100644 --- a/appengine/flexible/tasks/create_http_task_with_token.py +++ b/tasks/create_http_task_with_token.py @@ -37,7 +37,7 @@ def create_http_task(project, # project = 'my-project-id' # queue = 'my-appengine-queue' # location = 'us-central1' - # url = 'https://.appspot.com/example_task_handler' + # url = 'https://example.com/example_task_handler' # payload = 'hello' # Construct the fully qualified queue name. diff --git a/appengine/flexible/tasks/create_http_task_with_token_test.py b/tasks/create_http_task_with_token_test.py similarity index 93% rename from appengine/flexible/tasks/create_http_task_with_token_test.py rename to tasks/create_http_task_with_token_test.py index aa33b672ad..32cc4aa348 100644 --- a/appengine/flexible/tasks/create_http_task_with_token_test.py +++ b/tasks/create_http_task_with_token_test.py @@ -19,7 +19,8 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') -TEST_SERVICE_ACCOUNT = os.getenv('CLOUD_RUN_INVOKER_SERVICE_ACCOUNT') +TEST_SERVICE_ACCOUNT = 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com' + def test_create_http_task_with_token(): From ddb643fa548bd360bd61d6cc55506ab457e1c2e4 Mon Sep 17 00:00:00 2001 From: averikitsch Date: Thu, 18 Apr 2019 11:37:21 -0700 Subject: [PATCH 7/7] Update version and linting --- tasks/create_http_task_with_token_test.py | 4 ++-- tasks/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/create_http_task_with_token_test.py b/tasks/create_http_task_with_token_test.py index 32cc4aa348..5b98c566e9 100644 --- a/tasks/create_http_task_with_token_test.py +++ b/tasks/create_http_task_with_token_test.py @@ -19,8 +19,8 @@ TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'my-appengine-queue') -TEST_SERVICE_ACCOUNT = 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com' - +TEST_SERVICE_ACCOUNT = ( + 'test-run-invoker@python-docs-samples-tests.iam.gserviceaccount.com') def test_create_http_task_with_token(): diff --git a/tasks/requirements.txt b/tasks/requirements.txt index 9aadd8a774..fe50a5aa3b 100644 --- a/tasks/requirements.txt +++ b/tasks/requirements.txt @@ -1,3 +1,3 @@ Flask==1.0.2 gunicorn==19.9.0 -google-cloud-tasks==0.6.0 +google-cloud-tasks==0.7.0