11/19/2023 0 Comments Airflow dag bagTherefore only once per DagBag is a file logged This is to prevent overloading the user with logging Has_logged – an instance boolean that gets flipped from False to True after aįile has been skipped. Include_examples ( bool) – whether to include the examples that ship Settings are now dagbag level so that one system can run multiple,ĭag_folder ( unicode) – the folder to scan to find DAGsĮxecutor – the executor to use when executing task instances This makes it easier to runĭistinct environments for say production and development, tests, or forĭifferent teams or security profiles. Level configuration settings, like what database to use as a backend and DagBag ( dag_folder=None, executor=None, include_examples=conf.getboolean('core', 'LOAD_EXAMPLES'), safe_mode=conf.getboolean('core', 'DAG_DISCOVERY_SAFE_MODE'), store_serialized_dags=False ) ¶īases: _dag.BaseDagBag, _mixin.LoggingMixinĪ dagbag is a collection of dags, parsed out of a folder tree and has high Given a list of dag_ids, get string representing how close any that are dataset triggered are their next run, e.g. def get_dataset_triggered_next_run_info ( dag_ids : list, *, session : Session ) -> dict ]: """ Get next run info for a list of dag_ids. from _future_ import annotations import collections import collections.abc import copy import functools import itertools import logging import os import pathlib import pickle import sys import traceback import warnings import weakref from collections import deque from datetime import datetime, timedelta from inspect import signature from typing import ( TYPE_CHECKING, Any, Callable, Collection, Container, Iterable, Iterator, List, Pattern, Sequence, Union, cast, overload, ) from urllib.parse import urlsplit import jinja2 import pendulum import re2 from dateutil.relativedelta import relativedelta from pendulum.tz.timezone import Timezone from sqlalchemy import ( Boolean, Column, ForeignKey, Index, Integer, String, Text, and_, case, func, not_, or_, select, update, ) from import association_proxy from sqlalchemy.orm import backref, joinedload, relationship from import Query from import Session from sqlalchemy.sql import Select, expression import airflow.templates from airflow import settings, utils from airflow.api_internal.internal_api_call import internal_api_call from nfiguration import conf as airflow_conf, secrets_backend_list from airflow.exceptions import ( AirflowDagInconsistent, AirflowException, AirflowSkipException, DuplicateTaskIdFound, FailStopDagInvalidTriggerRule, RemovedInAirflow3Warning, TaskNotFound, ) from import run_job from import AbstractOperator from import Base, StringID from import BaseOperator from import DagCode from import DagPickle from import RUN_ID_REGEX, DagRun from import Operator from import DagParam, ParamsDict from import Context, TaskInstance, TaskInstanceKey, clear_task_instances from _filesystem import LocalFilesystemBackend from curity import permissions from airflow.stats import Stats from import DagRunInfo, DataInterval, TimeRestriction, Timetable from import CronDataIntervalTimetable, DeltaDataIntervalTimetable from import ( ContinuousTimetable, DatasetTriggeredTimetable, NullTimetable, OnceTimetable, ) from airflow.typing_compat import Literal from airflow.utils import timezone from _cycle_tester import check_cycle from import cron_presets, date_range as utils_date_range from import fixup_decorator_warning_stack from import at_most_one, exactly_one, validate_key from _mixin import LoggingMixin from import NEW_SESSION, provide_session from import ( Interval, UtcDateTime, lock_rows, skip_locked, tuple_in_condition, with_row_locks, ) from import DagRunState, TaskInstanceState from _rule import TriggerRule from import NOTSET, ArgNotSet, DagRunType, EdgeInfoType if TYPE_CHECKING : from types import ModuleType from airflow.datasets import Dataset from corators import TaskDecoratorCollection from import DagBag from import SlaMiss from _group import TaskGroup See the License for the # specific language governing permissions and limitations # under the License. You may obtain a copy of the License at # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License") you may not use this file except in compliance # with the License. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements.
0 Comments
Leave a Reply. |
AuthorWrite something about yourself. No need to be fancy, just an overview. ArchivesCategories |