Feat : DAG creation for all pipelines

GoogleCloudPlatform · nlarge-google · Aug 26, 2022 · Apr 16, 2022 · Aug 17, 2022 · Aug 18, 2022
commit 0d48434720065ba58a9150872224e1feb0ca06b1
diff --git a/datasets/nhtsa_traffic_fatalities/infra/nhtsa_traffic_fatalities_dataset.tf b/datasets/nhtsa_traffic_fatalities/infra/nhtsa_traffic_fatalities_dataset.tf
@@ -0,0 +1,25 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "google_bigquery_dataset" "nhtsa_traffic_fatalities" {
+  dataset_id = "nhtsa_traffic_fatalities"
+  project    = var.project_id
+}
+
+output "bigquery_dataset-nhtsa_traffic_fatalities-dataset_id" {
+  value = google_bigquery_dataset.nhtsa_traffic_fatalities.dataset_id
+}
diff --git a/datasets/nhtsa_traffic_fatalities/infra/provider.tf b/datasets/nhtsa_traffic_fatalities/infra/provider.tf
@@ -0,0 +1,28 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+provider "google" {
+  project                     = var.project_id
+  impersonate_service_account = var.impersonating_acct
+  region                      = var.region
+}
+
+data "google_client_openid_userinfo" "me" {}
+
+output "impersonating-account" {
+  value = data.google_client_openid_userinfo.me.email
+}
diff --git a/datasets/nhtsa_traffic_fatalities/infra/variables.tf b/datasets/nhtsa_traffic_fatalities/infra/variables.tf
@@ -0,0 +1,26 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+variable "project_id" {}
+variable "bucket_name_prefix" {}
+variable "impersonating_acct" {}
+variable "region" {}
+variable "env" {}
+variable "iam_policies" {
+  default = {}
+}
+
diff --git a/...ffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2016_2017_schema.json b/...ffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2016_2017_schema.json
@@ -578,37 +578,37 @@
 	{
 		"name": "ptrlr1vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr1vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	}
 ]
diff --git a/...a_traffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2018_schema.json b/...a_traffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2018_schema.json
@@ -584,37 +584,37 @@
 	{
 		"name": "ptrlr1vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr1vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	}
 ]
diff --git a/...a_traffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2019_schema.json b/...a_traffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2019_schema.json
@@ -578,37 +578,37 @@
 	{
 		"name": "ptrlr1vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr1vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	}
 ]
diff --git a/...a_traffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2020_schema.json b/...a_traffic_fatalities/pipelines/_images/nhtsa_traffic_fatalities_parkwork_2020_schema.json
@@ -542,145 +542,145 @@
 	{
 		"name": "ptrlr1vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr1vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vin",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3vinname",
 		"type": "string",
-		"description": " ",
+		"description": "This data element records the vehicle identification number (VIN) of any trailing units of a combination vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pvpicmake",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the make (manufacturer brand name) of this vehicle as per NHTSA vPIC submissions.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pvpicmakename",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the make (manufacturer brand name) of this vehicle as per NHTSA vPIC submissions.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pvpicmodel",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the model of this vehicle using NHTSA’s VIN decoder application, vPIC.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pvpicmodelname",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the model of this vehicle using NHTSA’s VIN decoder application, vPIC.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pvpicbodyclass",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies a classification of this vehicle based on its general body configuration, size, shape, doors, etc., as defined by the manufacturer.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pvpicbodyclassname",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies a classification of this vehicle based on its general body configuration, size, shape, doors, etc., as defined by the manufacturer.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "picfinalbody",
 		"type": "string",
-		"description": " ",
+		"description": "This element captures the completed/finished body class for an incomplete vehicle. An incomplete vehicle is completed by a final stage manufacturer. The intent of this data element is to capture the body class for incomplete vehicles when they are finished for road-use.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "picfinalbodyname",
 		"type": "string",
-		"description": " ",
+		"description": "This element captures the completed/finished body class for an incomplete vehicle. An incomplete vehicle is completed by a final stage manufacturer. The intent of this data element is to capture the body class for incomplete vehicles when they are finished for road-use.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pgvwr_from",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the range of gross vehicle weight rating of the power unit as identified by the manufacturer through the vehicle’s VIN submission. GVWR_FROM defines the lowest value and GVWR_TO defines the highest value for the range of the GVWR specified by the manufacturer as the recommended loaded weight for a vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pgvwr_fromname",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the range of gross vehicle weight rating of the power unit as identified by the manufacturer through the vehicle’s VIN submission. GVWR_FROM defines the lowest value and GVWR_TO defines the highest value for the range of the GVWR specified by the manufacturer as the recommended loaded weight for a vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pgvwr_to",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the range of gross vehicle weight rating of the power unit as identified by the manufacturer through the vehicle’s VIN submission. GVWR_FROM defines the lowest value and GVWR_TO defines the highest value for the range of the GVWR specified by the manufacturer as the recommended loaded weight for a vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "pgvwr_toname",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the range of gross vehicle weight rating of the power unit as identified by the manufacturer through the vehicle’s VIN submission. GVWR_FROM defines the lowest value and GVWR_TO defines the highest value for the range of the GVWR specified by the manufacturer as the recommended loaded weight for a vehicle.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr1gvwr",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the gross vehicle weight rating of any trailing units as identified by the manufacturer in the vehicle’s VIN.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr1gvwrname",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the gross vehicle weight rating of any trailing units as identified by the manufacturer in the vehicle’s VIN.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2gvwr",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the gross vehicle weight rating of any trailing units as identified by the manufacturer in the vehicle’s VIN.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr2gvwrname",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the gross vehicle weight rating of any trailing units as identified by the manufacturer in the vehicle’s VIN.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3gvwr",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the gross vehicle weight rating of any trailing units as identified by the manufacturer in the vehicle’s VIN.",
 		"mode": "NULLABLE"
 	},
 	{
 		"name": "ptrlr3gvwrname",
 		"type": "string",
-		"description": " ",
+		"description": "This element identifies the gross vehicle weight rating of any trailing units as identified by the manufacturer in the vehicle’s VIN.",
 		"mode": "NULLABLE"
 	}
 ]