Your IP : 3.138.174.45
# orm/strategies.py
# Copyright (C) 2005-2024 the SQLAlchemy authors and contributors
# <see AUTHORS file>
#
# This module is part of SQLAlchemy and is released under
# the MIT License: https://www.opensource.org/licenses/mit-license.php
# mypy: ignore-errors
"""sqlalchemy.orm.interfaces.LoaderStrategy
implementations, and related MapperOptions."""
from __future__ import annotations
import collections
import itertools
from typing import Any
from typing import Dict
from typing import Tuple
from typing import TYPE_CHECKING
from . import attributes
from . import exc as orm_exc
from . import interfaces
from . import loading
from . import path_registry
from . import properties
from . import query
from . import relationships
from . import unitofwork
from . import util as orm_util
from .base import _DEFER_FOR_STATE
from .base import _RAISE_FOR_STATE
from .base import _SET_DEFERRED_EXPIRED
from .base import ATTR_WAS_SET
from .base import LoaderCallableStatus
from .base import PASSIVE_OFF
from .base import PassiveFlag
from .context import _column_descriptions
from .context import ORMCompileState
from .context import ORMSelectCompileState
from .context import QueryContext
from .interfaces import LoaderStrategy
from .interfaces import StrategizedProperty
from .session import _state_session
from .state import InstanceState
from .strategy_options import Load
from .util import _none_set
from .util import AliasedClass
from .. import event
from .. import exc as sa_exc
from .. import inspect
from .. import log
from .. import sql
from .. import util
from ..sql import util as sql_util
from ..sql import visitors
from ..sql.selectable import LABEL_STYLE_TABLENAME_PLUS_COL
from ..sql.selectable import Select
if TYPE_CHECKING:
from .relationships import RelationshipProperty
from ..sql.elements import ColumnElement
def _register_attribute(
prop,
mapper,
useobject,
compare_function=None,
typecallable=None,
callable_=None,
proxy_property=None,
active_history=False,
impl_class=None,
**kw,
):
listen_hooks = []
uselist = useobject and prop.uselist
if useobject and prop.single_parent:
listen_hooks.append(single_parent_validator)
if prop.key in prop.parent.validators:
fn, opts = prop.parent.validators[prop.key]
listen_hooks.append(
lambda desc, prop: orm_util._validator_events(
desc, prop.key, fn, **opts
)
)
if useobject:
listen_hooks.append(unitofwork.track_cascade_events)
# need to assemble backref listeners
# after the singleparentvalidator, mapper validator
if useobject:
backref = prop.back_populates
if backref and prop._effective_sync_backref:
listen_hooks.append(
lambda desc, prop: attributes.backref_listeners(
desc, backref, uselist
)
)
# a single MapperProperty is shared down a class inheritance
# hierarchy, so we set up attribute instrumentation and backref event
# for each mapper down the hierarchy.
# typically, "mapper" is the same as prop.parent, due to the way
# the configure_mappers() process runs, however this is not strongly
# enforced, and in the case of a second configure_mappers() run the
# mapper here might not be prop.parent; also, a subclass mapper may
# be called here before a superclass mapper. That is, can't depend
# on mappers not already being set up so we have to check each one.
for m in mapper.self_and_descendants:
if prop is m._props.get(
prop.key
) and not m.class_manager._attr_has_impl(prop.key):
desc = attributes.register_attribute_impl(
m.class_,
prop.key,
parent_token=prop,
uselist=uselist,
compare_function=compare_function,
useobject=useobject,
trackparent=useobject
and (
prop.single_parent
or prop.direction is interfaces.ONETOMANY
),
typecallable=typecallable,
callable_=callable_,
active_history=active_history,
impl_class=impl_class,
send_modified_events=not useobject or not prop.viewonly,
doc=prop.doc,
**kw,
)
for hook in listen_hooks:
hook(desc, prop)
@properties.ColumnProperty.strategy_for(instrument=False, deferred=False)
class UninstrumentedColumnLoader(LoaderStrategy):
"""Represent a non-instrumented MapperProperty.
The polymorphic_on argument of mapper() often results in this,
if the argument is against the with_polymorphic selectable.
"""
__slots__ = ("columns",)
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
self.columns = self.parent_property.columns
def setup_query(
self,
compile_state,
query_entity,
path,
loadopt,
adapter,
column_collection=None,
**kwargs,
):
for c in self.columns:
if adapter:
c = adapter.columns[c]
compile_state._append_dedupe_col_collection(c, column_collection)
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
pass
@log.class_logger
@properties.ColumnProperty.strategy_for(instrument=True, deferred=False)
class ColumnLoader(LoaderStrategy):
"""Provide loading behavior for a :class:`.ColumnProperty`."""
__slots__ = "columns", "is_composite"
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
self.columns = self.parent_property.columns
self.is_composite = hasattr(self.parent_property, "composite_class")
def setup_query(
self,
compile_state,
query_entity,
path,
loadopt,
adapter,
column_collection,
memoized_populators,
check_for_adapt=False,
**kwargs,
):
for c in self.columns:
if adapter:
if check_for_adapt:
c = adapter.adapt_check_present(c)
if c is None:
return
else:
c = adapter.columns[c]
compile_state._append_dedupe_col_collection(c, column_collection)
fetch = self.columns[0]
if adapter:
fetch = adapter.columns[fetch]
if fetch is None:
# None happens here only for dml bulk_persistence cases
# when context.DMLReturningColFilter is used
return
memoized_populators[self.parent_property] = fetch
def init_class_attribute(self, mapper):
self.is_class_level = True
coltype = self.columns[0].type
# TODO: check all columns ? check for foreign key as well?
active_history = (
self.parent_property.active_history
or self.columns[0].primary_key
or (
mapper.version_id_col is not None
and mapper._columntoproperty.get(mapper.version_id_col, None)
is self.parent_property
)
)
_register_attribute(
self.parent_property,
mapper,
useobject=False,
compare_function=coltype.compare_values,
active_history=active_history,
)
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
# look through list of columns represented here
# to see which, if any, is present in the row.
for col in self.columns:
if adapter:
col = adapter.columns[col]
getter = result._getter(col, False)
if getter:
populators["quick"].append((self.key, getter))
break
else:
populators["expire"].append((self.key, True))
@log.class_logger
@properties.ColumnProperty.strategy_for(query_expression=True)
class ExpressionColumnLoader(ColumnLoader):
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
# compare to the "default" expression that is mapped in
# the column. If it's sql.null, we don't need to render
# unless an expr is passed in the options.
null = sql.null().label(None)
self._have_default_expression = any(
not c.compare(null) for c in self.parent_property.columns
)
def setup_query(
self,
compile_state,
query_entity,
path,
loadopt,
adapter,
column_collection,
memoized_populators,
**kwargs,
):
columns = None
if loadopt and loadopt._extra_criteria:
columns = loadopt._extra_criteria
elif self._have_default_expression:
columns = self.parent_property.columns
if columns is None:
return
for c in columns:
if adapter:
c = adapter.columns[c]
compile_state._append_dedupe_col_collection(c, column_collection)
fetch = columns[0]
if adapter:
fetch = adapter.columns[fetch]
if fetch is None:
# None is not expected to be the result of any
# adapter implementation here, however there may be theoretical
# usages of returning() with context.DMLReturningColFilter
return
memoized_populators[self.parent_property] = fetch
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
# look through list of columns represented here
# to see which, if any, is present in the row.
if loadopt and loadopt._extra_criteria:
columns = loadopt._extra_criteria
for col in columns:
if adapter:
col = adapter.columns[col]
getter = result._getter(col, False)
if getter:
populators["quick"].append((self.key, getter))
break
else:
populators["expire"].append((self.key, True))
def init_class_attribute(self, mapper):
self.is_class_level = True
_register_attribute(
self.parent_property,
mapper,
useobject=False,
compare_function=self.columns[0].type.compare_values,
accepts_scalar_loader=False,
)
@log.class_logger
@properties.ColumnProperty.strategy_for(deferred=True, instrument=True)
@properties.ColumnProperty.strategy_for(
deferred=True, instrument=True, raiseload=True
)
@properties.ColumnProperty.strategy_for(do_nothing=True)
class DeferredColumnLoader(LoaderStrategy):
"""Provide loading behavior for a deferred :class:`.ColumnProperty`."""
__slots__ = "columns", "group", "raiseload"
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
if hasattr(self.parent_property, "composite_class"):
raise NotImplementedError(
"Deferred loading for composite types not implemented yet"
)
self.raiseload = self.strategy_opts.get("raiseload", False)
self.columns = self.parent_property.columns
self.group = self.parent_property.group
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
# for a DeferredColumnLoader, this method is only used during a
# "row processor only" query; see test_deferred.py ->
# tests with "rowproc_only" in their name. As of the 1.0 series,
# loading._instance_processor doesn't use a "row processing" function
# to populate columns, instead it uses data in the "populators"
# dictionary. Normally, the DeferredColumnLoader.setup_query()
# sets up that data in the "memoized_populators" dictionary
# and "create_row_processor()" here is never invoked.
if (
context.refresh_state
and context.query._compile_options._only_load_props
and self.key in context.query._compile_options._only_load_props
):
self.parent_property._get_strategy(
(("deferred", False), ("instrument", True))
).create_row_processor(
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
)
elif not self.is_class_level:
if self.raiseload:
set_deferred_for_local_state = (
self.parent_property._raise_column_loader
)
else:
set_deferred_for_local_state = (
self.parent_property._deferred_column_loader
)
populators["new"].append((self.key, set_deferred_for_local_state))
else:
populators["expire"].append((self.key, False))
def init_class_attribute(self, mapper):
self.is_class_level = True
_register_attribute(
self.parent_property,
mapper,
useobject=False,
compare_function=self.columns[0].type.compare_values,
callable_=self._load_for_state,
load_on_unexpire=False,
)
def setup_query(
self,
compile_state,
query_entity,
path,
loadopt,
adapter,
column_collection,
memoized_populators,
only_load_props=None,
**kw,
):
if (
(
compile_state.compile_options._render_for_subquery
and self.parent_property._renders_in_subqueries
)
or (
loadopt
and set(self.columns).intersection(
self.parent._should_undefer_in_wildcard
)
)
or (
loadopt
and self.group
and loadopt.local_opts.get(
"undefer_group_%s" % self.group, False
)
)
or (only_load_props and self.key in only_load_props)
):
self.parent_property._get_strategy(
(("deferred", False), ("instrument", True))
).setup_query(
compile_state,
query_entity,
path,
loadopt,
adapter,
column_collection,
memoized_populators,
**kw,
)
elif self.is_class_level:
memoized_populators[self.parent_property] = _SET_DEFERRED_EXPIRED
elif not self.raiseload:
memoized_populators[self.parent_property] = _DEFER_FOR_STATE
else:
memoized_populators[self.parent_property] = _RAISE_FOR_STATE
def _load_for_state(self, state, passive):
if not state.key:
return LoaderCallableStatus.ATTR_EMPTY
if not passive & PassiveFlag.SQL_OK:
return LoaderCallableStatus.PASSIVE_NO_RESULT
localparent = state.manager.mapper
if self.group:
toload = [
p.key
for p in localparent.iterate_properties
if isinstance(p, StrategizedProperty)
and isinstance(p.strategy, DeferredColumnLoader)
and p.group == self.group
]
else:
toload = [self.key]
# narrow the keys down to just those which have no history
group = [k for k in toload if k in state.unmodified]
session = _state_session(state)
if session is None:
raise orm_exc.DetachedInstanceError(
"Parent instance %s is not bound to a Session; "
"deferred load operation of attribute '%s' cannot proceed"
% (orm_util.state_str(state), self.key)
)
if self.raiseload:
self._invoke_raise_load(state, passive, "raise")
loading.load_scalar_attributes(
state.mapper, state, set(group), PASSIVE_OFF
)
return LoaderCallableStatus.ATTR_WAS_SET
def _invoke_raise_load(self, state, passive, lazy):
raise sa_exc.InvalidRequestError(
"'%s' is not available due to raiseload=True" % (self,)
)
class LoadDeferredColumns:
"""serializable loader object used by DeferredColumnLoader"""
def __init__(self, key: str, raiseload: bool = False):
self.key = key
self.raiseload = raiseload
def __call__(self, state, passive=attributes.PASSIVE_OFF):
key = self.key
localparent = state.manager.mapper
prop = localparent._props[key]
if self.raiseload:
strategy_key = (
("deferred", True),
("instrument", True),
("raiseload", True),
)
else:
strategy_key = (("deferred", True), ("instrument", True))
strategy = prop._get_strategy(strategy_key)
return strategy._load_for_state(state, passive)
class AbstractRelationshipLoader(LoaderStrategy):
"""LoaderStratgies which deal with related objects."""
__slots__ = "mapper", "target", "uselist", "entity"
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
self.mapper = self.parent_property.mapper
self.entity = self.parent_property.entity
self.target = self.parent_property.target
self.uselist = self.parent_property.uselist
def _immediateload_create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
return self.parent_property._get_strategy(
(("lazy", "immediate"),)
).create_row_processor(
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
)
@log.class_logger
@relationships.RelationshipProperty.strategy_for(do_nothing=True)
class DoNothingLoader(LoaderStrategy):
"""Relationship loader that makes no change to the object's state.
Compared to NoLoader, this loader does not initialize the
collection/attribute to empty/none; the usual default LazyLoader will
take effect.
"""
@log.class_logger
@relationships.RelationshipProperty.strategy_for(lazy="noload")
@relationships.RelationshipProperty.strategy_for(lazy=None)
class NoLoader(AbstractRelationshipLoader):
"""Provide loading behavior for a :class:`.Relationship`
with "lazy=None".
"""
__slots__ = ()
def init_class_attribute(self, mapper):
self.is_class_level = True
_register_attribute(
self.parent_property,
mapper,
useobject=True,
typecallable=self.parent_property.collection_class,
)
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
def invoke_no_load(state, dict_, row):
if self.uselist:
attributes.init_state_collection(state, dict_, self.key)
else:
dict_[self.key] = None
populators["new"].append((self.key, invoke_no_load))
@log.class_logger
@relationships.RelationshipProperty.strategy_for(lazy=True)
@relationships.RelationshipProperty.strategy_for(lazy="select")
@relationships.RelationshipProperty.strategy_for(lazy="raise")
@relationships.RelationshipProperty.strategy_for(lazy="raise_on_sql")
@relationships.RelationshipProperty.strategy_for(lazy="baked_select")
class LazyLoader(
AbstractRelationshipLoader, util.MemoizedSlots, log.Identified
):
"""Provide loading behavior for a :class:`.Relationship`
with "lazy=True", that is loads when first accessed.
"""
__slots__ = (
"_lazywhere",
"_rev_lazywhere",
"_lazyload_reverse_option",
"_order_by",
"use_get",
"is_aliased_class",
"_bind_to_col",
"_equated_columns",
"_rev_bind_to_col",
"_rev_equated_columns",
"_simple_lazy_clause",
"_raise_always",
"_raise_on_sql",
)
_lazywhere: ColumnElement[bool]
_bind_to_col: Dict[str, ColumnElement[Any]]
_rev_lazywhere: ColumnElement[bool]
_rev_bind_to_col: Dict[str, ColumnElement[Any]]
parent_property: RelationshipProperty[Any]
def __init__(
self, parent: RelationshipProperty[Any], strategy_key: Tuple[Any, ...]
):
super().__init__(parent, strategy_key)
self._raise_always = self.strategy_opts["lazy"] == "raise"
self._raise_on_sql = self.strategy_opts["lazy"] == "raise_on_sql"
self.is_aliased_class = inspect(self.entity).is_aliased_class
join_condition = self.parent_property._join_condition
(
self._lazywhere,
self._bind_to_col,
self._equated_columns,
) = join_condition.create_lazy_clause()
(
self._rev_lazywhere,
self._rev_bind_to_col,
self._rev_equated_columns,
) = join_condition.create_lazy_clause(reverse_direction=True)
if self.parent_property.order_by:
self._order_by = [
sql_util._deep_annotate(elem, {"_orm_adapt": True})
for elem in util.to_list(self.parent_property.order_by)
]
else:
self._order_by = None
self.logger.info("%s lazy loading clause %s", self, self._lazywhere)
# determine if our "lazywhere" clause is the same as the mapper's
# get() clause. then we can just use mapper.get()
#
# TODO: the "not self.uselist" can be taken out entirely; a m2o
# load that populates for a list (very unusual, but is possible with
# the API) can still set for "None" and the attribute system will
# populate as an empty list.
self.use_get = (
not self.is_aliased_class
and not self.uselist
and self.entity._get_clause[0].compare(
self._lazywhere,
use_proxies=True,
compare_keys=False,
equivalents=self.mapper._equivalent_columns,
)
)
if self.use_get:
for col in list(self._equated_columns):
if col in self.mapper._equivalent_columns:
for c in self.mapper._equivalent_columns[col]:
self._equated_columns[c] = self._equated_columns[col]
self.logger.info(
"%s will use Session.get() to optimize instance loads", self
)
def init_class_attribute(self, mapper):
self.is_class_level = True
_legacy_inactive_history_style = (
self.parent_property._legacy_inactive_history_style
)
if self.parent_property.active_history:
active_history = True
_deferred_history = False
elif (
self.parent_property.direction is not interfaces.MANYTOONE
or not self.use_get
):
if _legacy_inactive_history_style:
active_history = True
_deferred_history = False
else:
active_history = False
_deferred_history = True
else:
active_history = _deferred_history = False
_register_attribute(
self.parent_property,
mapper,
useobject=True,
callable_=self._load_for_state,
typecallable=self.parent_property.collection_class,
active_history=active_history,
_deferred_history=_deferred_history,
)
def _memoized_attr__simple_lazy_clause(self):
lazywhere = sql_util._deep_annotate(
self._lazywhere, {"_orm_adapt": True}
)
criterion, bind_to_col = (lazywhere, self._bind_to_col)
params = []
def visit_bindparam(bindparam):
bindparam.unique = False
visitors.traverse(criterion, {}, {"bindparam": visit_bindparam})
def visit_bindparam(bindparam):
if bindparam._identifying_key in bind_to_col:
params.append(
(
bindparam.key,
bind_to_col[bindparam._identifying_key],
None,
)
)
elif bindparam.callable is None:
params.append((bindparam.key, None, bindparam.value))
criterion = visitors.cloned_traverse(
criterion, {}, {"bindparam": visit_bindparam}
)
return criterion, params
def _generate_lazy_clause(self, state, passive):
criterion, param_keys = self._simple_lazy_clause
if state is None:
return sql_util.adapt_criterion_to_null(
criterion, [key for key, ident, value in param_keys]
)
mapper = self.parent_property.parent
o = state.obj() # strong ref
dict_ = attributes.instance_dict(o)
if passive & PassiveFlag.INIT_OK:
passive ^= PassiveFlag.INIT_OK
params = {}
for key, ident, value in param_keys:
if ident is not None:
if passive and passive & PassiveFlag.LOAD_AGAINST_COMMITTED:
value = mapper._get_committed_state_attr_by_column(
state, dict_, ident, passive
)
else:
value = mapper._get_state_attr_by_column(
state, dict_, ident, passive
)
params[key] = value
return criterion, params
def _invoke_raise_load(self, state, passive, lazy):
raise sa_exc.InvalidRequestError(
"'%s' is not available due to lazy='%s'" % (self, lazy)
)
def _load_for_state(
self,
state,
passive,
loadopt=None,
extra_criteria=(),
extra_options=(),
alternate_effective_path=None,
execution_options=util.EMPTY_DICT,
):
if not state.key and (
(
not self.parent_property.load_on_pending
and not state._load_pending
)
or not state.session_id
):
return LoaderCallableStatus.ATTR_EMPTY
pending = not state.key
primary_key_identity = None
use_get = self.use_get and (not loadopt or not loadopt._extra_criteria)
if (not passive & PassiveFlag.SQL_OK and not use_get) or (
not passive & attributes.NON_PERSISTENT_OK and pending
):
return LoaderCallableStatus.PASSIVE_NO_RESULT
if (
# we were given lazy="raise"
self._raise_always
# the no_raise history-related flag was not passed
and not passive & PassiveFlag.NO_RAISE
and (
# if we are use_get and related_object_ok is disabled,
# which means we are at most looking in the identity map
# for history purposes or otherwise returning
# PASSIVE_NO_RESULT, don't raise. This is also a
# history-related flag
not use_get
or passive & PassiveFlag.RELATED_OBJECT_OK
)
):
self._invoke_raise_load(state, passive, "raise")
session = _state_session(state)
if not session:
if passive & PassiveFlag.NO_RAISE:
return LoaderCallableStatus.PASSIVE_NO_RESULT
raise orm_exc.DetachedInstanceError(
"Parent instance %s is not bound to a Session; "
"lazy load operation of attribute '%s' cannot proceed"
% (orm_util.state_str(state), self.key)
)
# if we have a simple primary key load, check the
# identity map without generating a Query at all
if use_get:
primary_key_identity = self._get_ident_for_use_get(
session, state, passive
)
if LoaderCallableStatus.PASSIVE_NO_RESULT in primary_key_identity:
return LoaderCallableStatus.PASSIVE_NO_RESULT
elif LoaderCallableStatus.NEVER_SET in primary_key_identity:
return LoaderCallableStatus.NEVER_SET
if _none_set.issuperset(primary_key_identity):
return None
if (
self.key in state.dict
and not passive & PassiveFlag.DEFERRED_HISTORY_LOAD
):
return LoaderCallableStatus.ATTR_WAS_SET
# look for this identity in the identity map. Delegate to the
# Query class in use, as it may have special rules for how it
# does this, including how it decides what the correct
# identity_token would be for this identity.
instance = session._identity_lookup(
self.entity,
primary_key_identity,
passive=passive,
lazy_loaded_from=state,
)
if instance is not None:
if instance is LoaderCallableStatus.PASSIVE_CLASS_MISMATCH:
return None
else:
return instance
elif (
not passive & PassiveFlag.SQL_OK
or not passive & PassiveFlag.RELATED_OBJECT_OK
):
return LoaderCallableStatus.PASSIVE_NO_RESULT
return self._emit_lazyload(
session,
state,
primary_key_identity,
passive,
loadopt,
extra_criteria,
extra_options,
alternate_effective_path,
execution_options,
)
def _get_ident_for_use_get(self, session, state, passive):
instance_mapper = state.manager.mapper
if passive & PassiveFlag.LOAD_AGAINST_COMMITTED:
get_attr = instance_mapper._get_committed_state_attr_by_column
else:
get_attr = instance_mapper._get_state_attr_by_column
dict_ = state.dict
return [
get_attr(state, dict_, self._equated_columns[pk], passive=passive)
for pk in self.mapper.primary_key
]
@util.preload_module("sqlalchemy.orm.strategy_options")
def _emit_lazyload(
self,
session,
state,
primary_key_identity,
passive,
loadopt,
extra_criteria,
extra_options,
alternate_effective_path,
execution_options,
):
strategy_options = util.preloaded.orm_strategy_options
clauseelement = self.entity.__clause_element__()
stmt = Select._create_raw_select(
_raw_columns=[clauseelement],
_propagate_attrs=clauseelement._propagate_attrs,
_label_style=LABEL_STYLE_TABLENAME_PLUS_COL,
_compile_options=ORMCompileState.default_compile_options,
)
load_options = QueryContext.default_load_options
load_options += {
"_invoke_all_eagers": False,
"_lazy_loaded_from": state,
}
if self.parent_property.secondary is not None:
stmt = stmt.select_from(
self.mapper, self.parent_property.secondary
)
pending = not state.key
# don't autoflush on pending
if pending or passive & attributes.NO_AUTOFLUSH:
stmt._execution_options = util.immutabledict({"autoflush": False})
use_get = self.use_get
if state.load_options or (loadopt and loadopt._extra_criteria):
if alternate_effective_path is None:
effective_path = state.load_path[self.parent_property]
else:
effective_path = alternate_effective_path[self.parent_property]
opts = state.load_options
if loadopt and loadopt._extra_criteria:
use_get = False
opts += (
orm_util.LoaderCriteriaOption(self.entity, extra_criteria),
)
stmt._with_options = opts
elif alternate_effective_path is None:
# this path is used if there are not already any options
# in the query, but an event may want to add them
effective_path = state.mapper._path_registry[self.parent_property]
else:
# added by immediateloader
effective_path = alternate_effective_path[self.parent_property]
if extra_options:
stmt._with_options += extra_options
stmt._compile_options += {"_current_path": effective_path}
if use_get:
if self._raise_on_sql and not passive & PassiveFlag.NO_RAISE:
self._invoke_raise_load(state, passive, "raise_on_sql")
return loading.load_on_pk_identity(
session,
stmt,
primary_key_identity,
load_options=load_options,
execution_options=execution_options,
)
if self._order_by:
stmt._order_by_clauses = self._order_by
def _lazyload_reverse(compile_context):
for rev in self.parent_property._reverse_property:
# reverse props that are MANYTOONE are loading *this*
# object from get(), so don't need to eager out to those.
if (
rev.direction is interfaces.MANYTOONE
and rev._use_get
and not isinstance(rev.strategy, LazyLoader)
):
strategy_options.Load._construct_for_existing_path(
compile_context.compile_options._current_path[
rev.parent
]
).lazyload(rev).process_compile_state(compile_context)
stmt._with_context_options += (
(_lazyload_reverse, self.parent_property),
)
lazy_clause, params = self._generate_lazy_clause(state, passive)
if execution_options:
execution_options = util.EMPTY_DICT.merge_with(
execution_options,
{
"_sa_orm_load_options": load_options,
},
)
else:
execution_options = {
"_sa_orm_load_options": load_options,
}
if (
self.key in state.dict
and not passive & PassiveFlag.DEFERRED_HISTORY_LOAD
):
return LoaderCallableStatus.ATTR_WAS_SET
if pending:
if util.has_intersection(orm_util._none_set, params.values()):
return None
elif util.has_intersection(orm_util._never_set, params.values()):
return None
if self._raise_on_sql and not passive & PassiveFlag.NO_RAISE:
self._invoke_raise_load(state, passive, "raise_on_sql")
stmt._where_criteria = (lazy_clause,)
result = session.execute(
stmt, params, execution_options=execution_options
)
result = result.unique().scalars().all()
if self.uselist:
return result
else:
l = len(result)
if l:
if l > 1:
util.warn(
"Multiple rows returned with "
"uselist=False for lazily-loaded attribute '%s' "
% self.parent_property
)
return result[0]
else:
return None
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
key = self.key
if (
context.load_options._is_user_refresh
and context.query._compile_options._only_load_props
and self.key in context.query._compile_options._only_load_props
):
return self._immediateload_create_row_processor(
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
)
if not self.is_class_level or (loadopt and loadopt._extra_criteria):
# we are not the primary manager for this attribute
# on this class - set up a
# per-instance lazyloader, which will override the
# class-level behavior.
# this currently only happens when using a
# "lazyload" option on a "no load"
# attribute - "eager" attributes always have a
# class-level lazyloader installed.
set_lazy_callable = (
InstanceState._instance_level_callable_processor
)(
mapper.class_manager,
LoadLazyAttribute(
key,
self,
loadopt,
(
loadopt._generate_extra_criteria(context)
if loadopt._extra_criteria
else None
),
),
key,
)
populators["new"].append((self.key, set_lazy_callable))
elif context.populate_existing or mapper.always_refresh:
def reset_for_lazy_callable(state, dict_, row):
# we are the primary manager for this attribute on
# this class - reset its
# per-instance attribute state, so that the class-level
# lazy loader is
# executed when next referenced on this instance.
# this is needed in
# populate_existing() types of scenarios to reset
# any existing state.
state._reset(dict_, key)
populators["new"].append((self.key, reset_for_lazy_callable))
class LoadLazyAttribute:
"""semi-serializable loader object used by LazyLoader
Historically, this object would be carried along with instances that
needed to run lazyloaders, so it had to be serializable to support
cached instances.
this is no longer a general requirement, and the case where this object
is used is exactly the case where we can't really serialize easily,
which is when extra criteria in the loader option is present.
We can't reliably serialize that as it refers to mapped entities and
AliasedClass objects that are local to the current process, which would
need to be matched up on deserialize e.g. the sqlalchemy.ext.serializer
approach.
"""
def __init__(self, key, initiating_strategy, loadopt, extra_criteria):
self.key = key
self.strategy_key = initiating_strategy.strategy_key
self.loadopt = loadopt
self.extra_criteria = extra_criteria
def __getstate__(self):
if self.extra_criteria is not None:
util.warn(
"Can't reliably serialize a lazyload() option that "
"contains additional criteria; please use eager loading "
"for this case"
)
return {
"key": self.key,
"strategy_key": self.strategy_key,
"loadopt": self.loadopt,
"extra_criteria": (),
}
def __call__(self, state, passive=attributes.PASSIVE_OFF):
key = self.key
instance_mapper = state.manager.mapper
prop = instance_mapper._props[key]
strategy = prop._strategies[self.strategy_key]
return strategy._load_for_state(
state,
passive,
loadopt=self.loadopt,
extra_criteria=self.extra_criteria,
)
class PostLoader(AbstractRelationshipLoader):
"""A relationship loader that emits a second SELECT statement."""
__slots__ = ()
def _setup_for_recursion(self, context, path, loadopt, join_depth=None):
effective_path = (
context.compile_state.current_path or orm_util.PathRegistry.root
) + path
top_level_context = context._get_top_level_context()
execution_options = util.immutabledict(
{"sa_top_level_orm_context": top_level_context}
)
if loadopt:
recursion_depth = loadopt.local_opts.get("recursion_depth", None)
unlimited_recursion = recursion_depth == -1
else:
recursion_depth = None
unlimited_recursion = False
if recursion_depth is not None:
if not self.parent_property._is_self_referential:
raise sa_exc.InvalidRequestError(
f"recursion_depth option on relationship "
f"{self.parent_property} not valid for "
"non-self-referential relationship"
)
recursion_depth = context.execution_options.get(
f"_recursion_depth_{id(self)}", recursion_depth
)
if not unlimited_recursion and recursion_depth < 0:
return (
effective_path,
False,
execution_options,
recursion_depth,
)
if not unlimited_recursion:
execution_options = execution_options.union(
{
f"_recursion_depth_{id(self)}": recursion_depth - 1,
}
)
if loading.PostLoad.path_exists(
context, effective_path, self.parent_property
):
return effective_path, False, execution_options, recursion_depth
path_w_prop = path[self.parent_property]
effective_path_w_prop = effective_path[self.parent_property]
if not path_w_prop.contains(context.attributes, "loader"):
if join_depth:
if effective_path_w_prop.length / 2 > join_depth:
return (
effective_path,
False,
execution_options,
recursion_depth,
)
elif effective_path_w_prop.contains_mapper(self.mapper):
return (
effective_path,
False,
execution_options,
recursion_depth,
)
return effective_path, True, execution_options, recursion_depth
@relationships.RelationshipProperty.strategy_for(lazy="immediate")
class ImmediateLoader(PostLoader):
__slots__ = ("join_depth",)
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
self.join_depth = self.parent_property.join_depth
def init_class_attribute(self, mapper):
self.parent_property._get_strategy(
(("lazy", "select"),)
).init_class_attribute(mapper)
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
if not context.compile_state.compile_options._enable_eagerloads:
return
(
effective_path,
run_loader,
execution_options,
recursion_depth,
) = self._setup_for_recursion(context, path, loadopt, self.join_depth)
if not run_loader:
# this will not emit SQL and will only emit for a many-to-one
# "use get" load. the "_RELATED" part means it may return
# instance even if its expired, since this is a mutually-recursive
# load operation.
flags = attributes.PASSIVE_NO_FETCH_RELATED | PassiveFlag.NO_RAISE
else:
flags = attributes.PASSIVE_OFF | PassiveFlag.NO_RAISE
loading.PostLoad.callable_for_path(
context,
effective_path,
self.parent,
self.parent_property,
self._load_for_path,
loadopt,
flags,
recursion_depth,
execution_options,
)
def _load_for_path(
self,
context,
path,
states,
load_only,
loadopt,
flags,
recursion_depth,
execution_options,
):
if recursion_depth:
new_opt = Load(loadopt.path.entity)
new_opt.context = (
loadopt,
loadopt._recurse(),
)
alternate_effective_path = path._truncate_recursive()
extra_options = (new_opt,)
else:
new_opt = None
alternate_effective_path = path
extra_options = ()
key = self.key
lazyloader = self.parent_property._get_strategy((("lazy", "select"),))
for state, overwrite in states:
dict_ = state.dict
if overwrite or key not in dict_:
value = lazyloader._load_for_state(
state,
flags,
extra_options=extra_options,
alternate_effective_path=alternate_effective_path,
execution_options=execution_options,
)
if value not in (
ATTR_WAS_SET,
LoaderCallableStatus.PASSIVE_NO_RESULT,
):
state.get_impl(key).set_committed_value(
state, dict_, value
)
@log.class_logger
@relationships.RelationshipProperty.strategy_for(lazy="subquery")
class SubqueryLoader(PostLoader):
__slots__ = ("join_depth",)
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
self.join_depth = self.parent_property.join_depth
def init_class_attribute(self, mapper):
self.parent_property._get_strategy(
(("lazy", "select"),)
).init_class_attribute(mapper)
def _get_leftmost(
self,
orig_query_entity_index,
subq_path,
current_compile_state,
is_root,
):
given_subq_path = subq_path
subq_path = subq_path.path
subq_mapper = orm_util._class_to_mapper(subq_path[0])
# determine attributes of the leftmost mapper
if (
self.parent.isa(subq_mapper)
and self.parent_property is subq_path[1]
):
leftmost_mapper, leftmost_prop = self.parent, self.parent_property
else:
leftmost_mapper, leftmost_prop = subq_mapper, subq_path[1]
if is_root:
# the subq_path is also coming from cached state, so when we start
# building up this path, it has to also be converted to be in terms
# of the current state. this is for the specific case of the entity
# is an AliasedClass against a subquery that's not otherwise going
# to adapt
new_subq_path = current_compile_state._entities[
orig_query_entity_index
].entity_zero._path_registry[leftmost_prop]
additional = len(subq_path) - len(new_subq_path)
if additional:
new_subq_path += path_registry.PathRegistry.coerce(
subq_path[-additional:]
)
else:
new_subq_path = given_subq_path
leftmost_cols = leftmost_prop.local_columns
leftmost_attr = [
getattr(
new_subq_path.path[0].entity,
leftmost_mapper._columntoproperty[c].key,
)
for c in leftmost_cols
]
return leftmost_mapper, leftmost_attr, leftmost_prop, new_subq_path
def _generate_from_original_query(
self,
orig_compile_state,
orig_query,
leftmost_mapper,
leftmost_attr,
leftmost_relationship,
orig_entity,
):
# reformat the original query
# to look only for significant columns
q = orig_query._clone().correlate(None)
# LEGACY: make a Query back from the select() !!
# This suits at least two legacy cases:
# 1. applications which expect before_compile() to be called
# below when we run .subquery() on this query (Keystone)
# 2. applications which are doing subqueryload with complex
# from_self() queries, as query.subquery() / .statement
# has to do the full compile context for multiply-nested
# from_self() (Neutron) - see test_subqload_from_self
# for demo.
q2 = query.Query.__new__(query.Query)
q2.__dict__.update(q.__dict__)
q = q2
# set the query's "FROM" list explicitly to what the
# FROM list would be in any case, as we will be limiting
# the columns in the SELECT list which may no longer include
# all entities mentioned in things like WHERE, JOIN, etc.
if not q._from_obj:
q._enable_assertions = False
q.select_from.non_generative(
q,
*{
ent["entity"]
for ent in _column_descriptions(
orig_query, compile_state=orig_compile_state
)
if ent["entity"] is not None
},
)
# select from the identity columns of the outer (specifically, these
# are the 'local_cols' of the property). This will remove other
# columns from the query that might suggest the right entity which is
# why we do set select_from above. The attributes we have are
# coerced and adapted using the original query's adapter, which is
# needed only for the case of adapting a subclass column to
# that of a polymorphic selectable, e.g. we have
# Engineer.primary_language and the entity is Person. All other
# adaptations, e.g. from_self, select_entity_from(), will occur
# within the new query when it compiles, as the compile_state we are
# using here is only a partial one. If the subqueryload is from a
# with_polymorphic() or other aliased() object, left_attr will already
# be the correct attributes so no adaptation is needed.
target_cols = orig_compile_state._adapt_col_list(
[
sql.coercions.expect(sql.roles.ColumnsClauseRole, o)
for o in leftmost_attr
],
orig_compile_state._get_current_adapter(),
)
q._raw_columns = target_cols
distinct_target_key = leftmost_relationship.distinct_target_key
if distinct_target_key is True:
q._distinct = True
elif distinct_target_key is None:
# if target_cols refer to a non-primary key or only
# part of a composite primary key, set the q as distinct
for t in {c.table for c in target_cols}:
if not set(target_cols).issuperset(t.primary_key):
q._distinct = True
break
# don't need ORDER BY if no limit/offset
if not q._has_row_limiting_clause:
q._order_by_clauses = ()
if q._distinct is True and q._order_by_clauses:
# the logic to automatically add the order by columns to the query
# when distinct is True is deprecated in the query
to_add = sql_util.expand_column_list_from_order_by(
target_cols, q._order_by_clauses
)
if to_add:
q._set_entities(target_cols + to_add)
# the original query now becomes a subquery
# which we'll join onto.
# LEGACY: as "q" is a Query, the before_compile() event is invoked
# here.
embed_q = q.set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL).subquery()
left_alias = orm_util.AliasedClass(
leftmost_mapper, embed_q, use_mapper_path=True
)
return left_alias
def _prep_for_joins(self, left_alias, subq_path):
# figure out what's being joined. a.k.a. the fun part
to_join = []
pairs = list(subq_path.pairs())
for i, (mapper, prop) in enumerate(pairs):
if i > 0:
# look at the previous mapper in the chain -
# if it is as or more specific than this prop's
# mapper, use that instead.
# note we have an assumption here that
# the non-first element is always going to be a mapper,
# not an AliasedClass
prev_mapper = pairs[i - 1][1].mapper
to_append = prev_mapper if prev_mapper.isa(mapper) else mapper
else:
to_append = mapper
to_join.append((to_append, prop.key))
# determine the immediate parent class we are joining from,
# which needs to be aliased.
if len(to_join) < 2:
# in the case of a one level eager load, this is the
# leftmost "left_alias".
parent_alias = left_alias
else:
info = inspect(to_join[-1][0])
if info.is_aliased_class:
parent_alias = info.entity
else:
# alias a plain mapper as we may be
# joining multiple times
parent_alias = orm_util.AliasedClass(
info.entity, use_mapper_path=True
)
local_cols = self.parent_property.local_columns
local_attr = [
getattr(parent_alias, self.parent._columntoproperty[c].key)
for c in local_cols
]
return to_join, local_attr, parent_alias
def _apply_joins(
self, q, to_join, left_alias, parent_alias, effective_entity
):
ltj = len(to_join)
if ltj == 1:
to_join = [
getattr(left_alias, to_join[0][1]).of_type(effective_entity)
]
elif ltj == 2:
to_join = [
getattr(left_alias, to_join[0][1]).of_type(parent_alias),
getattr(parent_alias, to_join[-1][1]).of_type(
effective_entity
),
]
elif ltj > 2:
middle = [
(
(
orm_util.AliasedClass(item[0])
if not inspect(item[0]).is_aliased_class
else item[0].entity
),
item[1],
)
for item in to_join[1:-1]
]
inner = []
while middle:
item = middle.pop(0)
attr = getattr(item[0], item[1])
if middle:
attr = attr.of_type(middle[0][0])
else:
attr = attr.of_type(parent_alias)
inner.append(attr)
to_join = (
[getattr(left_alias, to_join[0][1]).of_type(inner[0].parent)]
+ inner
+ [
getattr(parent_alias, to_join[-1][1]).of_type(
effective_entity
)
]
)
for attr in to_join:
q = q.join(attr)
return q
def _setup_options(
self,
context,
q,
subq_path,
rewritten_path,
orig_query,
effective_entity,
loadopt,
):
# note that because the subqueryload object
# does not re-use the cached query, instead always making
# use of the current invoked query, while we have two queries
# here (orig and context.query), they are both non-cached
# queries and we can transfer the options as is without
# adjusting for new criteria. Some work on #6881 / #6889
# brought this into question.
new_options = orig_query._with_options
if loadopt and loadopt._extra_criteria:
new_options += (
orm_util.LoaderCriteriaOption(
self.entity,
loadopt._generate_extra_criteria(context),
),
)
# propagate loader options etc. to the new query.
# these will fire relative to subq_path.
q = q._with_current_path(rewritten_path)
q = q.options(*new_options)
return q
def _setup_outermost_orderby(self, q):
if self.parent_property.order_by:
def _setup_outermost_orderby(compile_context):
compile_context.eager_order_by += tuple(
util.to_list(self.parent_property.order_by)
)
q = q._add_context_option(
_setup_outermost_orderby, self.parent_property
)
return q
class _SubqCollections:
"""Given a :class:`_query.Query` used to emit the "subquery load",
provide a load interface that executes the query at the
first moment a value is needed.
"""
__slots__ = (
"session",
"execution_options",
"load_options",
"params",
"subq",
"_data",
)
def __init__(self, context, subq):
# avoid creating a cycle by storing context
# even though that's preferable
self.session = context.session
self.execution_options = context.execution_options
self.load_options = context.load_options
self.params = context.params or {}
self.subq = subq
self._data = None
def get(self, key, default):
if self._data is None:
self._load()
return self._data.get(key, default)
def _load(self):
self._data = collections.defaultdict(list)
q = self.subq
assert q.session is None
q = q.with_session(self.session)
if self.load_options._populate_existing:
q = q.populate_existing()
# to work with baked query, the parameters may have been
# updated since this query was created, so take these into account
rows = list(q.params(self.params))
for k, v in itertools.groupby(rows, lambda x: x[1:]):
self._data[k].extend(vv[0] for vv in v)
def loader(self, state, dict_, row):
if self._data is None:
self._load()
def _setup_query_from_rowproc(
self,
context,
query_entity,
path,
entity,
loadopt,
adapter,
):
compile_state = context.compile_state
if (
not compile_state.compile_options._enable_eagerloads
or compile_state.compile_options._for_refresh_state
):
return
orig_query_entity_index = compile_state._entities.index(query_entity)
context.loaders_require_buffering = True
path = path[self.parent_property]
# build up a path indicating the path from the leftmost
# entity to the thing we're subquery loading.
with_poly_entity = path.get(
compile_state.attributes, "path_with_polymorphic", None
)
if with_poly_entity is not None:
effective_entity = with_poly_entity
else:
effective_entity = self.entity
subq_path, rewritten_path = context.query._execution_options.get(
("subquery_paths", None),
(orm_util.PathRegistry.root, orm_util.PathRegistry.root),
)
is_root = subq_path is orm_util.PathRegistry.root
subq_path = subq_path + path
rewritten_path = rewritten_path + path
# use the current query being invoked, not the compile state
# one. this is so that we get the current parameters. however,
# it means we can't use the existing compile state, we have to make
# a new one. other approaches include possibly using the
# compiled query but swapping the params, seems only marginally
# less time spent but more complicated
orig_query = context.query._execution_options.get(
("orig_query", SubqueryLoader), context.query
)
# make a new compile_state for the query that's probably cached, but
# we're sort of undoing a bit of that caching :(
compile_state_cls = ORMCompileState._get_plugin_class_for_plugin(
orig_query, "orm"
)
if orig_query._is_lambda_element:
if context.load_options._lazy_loaded_from is None:
util.warn(
'subqueryloader for "%s" must invoke lambda callable '
"at %r in "
"order to produce a new query, decreasing the efficiency "
"of caching for this statement. Consider using "
"selectinload() for more effective full-lambda caching"
% (self, orig_query)
)
orig_query = orig_query._resolved
# this is the more "quick" version, however it's not clear how
# much of this we need. in particular I can't get a test to
# fail if the "set_base_alias" is missing and not sure why that is.
orig_compile_state = compile_state_cls._create_entities_collection(
orig_query, legacy=False
)
(
leftmost_mapper,
leftmost_attr,
leftmost_relationship,
rewritten_path,
) = self._get_leftmost(
orig_query_entity_index,
rewritten_path,
orig_compile_state,
is_root,
)
# generate a new Query from the original, then
# produce a subquery from it.
left_alias = self._generate_from_original_query(
orig_compile_state,
orig_query,
leftmost_mapper,
leftmost_attr,
leftmost_relationship,
entity,
)
# generate another Query that will join the
# left alias to the target relationships.
# basically doing a longhand
# "from_self()". (from_self() itself not quite industrial
# strength enough for all contingencies...but very close)
q = query.Query(effective_entity)
q._execution_options = context.query._execution_options.merge_with(
context.execution_options,
{
("orig_query", SubqueryLoader): orig_query,
("subquery_paths", None): (subq_path, rewritten_path),
},
)
q = q._set_enable_single_crit(False)
to_join, local_attr, parent_alias = self._prep_for_joins(
left_alias, subq_path
)
q = q.add_columns(*local_attr)
q = self._apply_joins(
q, to_join, left_alias, parent_alias, effective_entity
)
q = self._setup_options(
context,
q,
subq_path,
rewritten_path,
orig_query,
effective_entity,
loadopt,
)
q = self._setup_outermost_orderby(q)
return q
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
if context.refresh_state:
return self._immediateload_create_row_processor(
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
)
_, run_loader, _, _ = self._setup_for_recursion(
context, path, loadopt, self.join_depth
)
if not run_loader:
return
if not isinstance(context.compile_state, ORMSelectCompileState):
# issue 7505 - subqueryload() in 1.3 and previous would silently
# degrade for from_statement() without warning. this behavior
# is restored here
return
if not self.parent.class_manager[self.key].impl.supports_population:
raise sa_exc.InvalidRequestError(
"'%s' does not support object "
"population - eager loading cannot be applied." % self
)
# a little dance here as the "path" is still something that only
# semi-tracks the exact series of things we are loading, still not
# telling us about with_polymorphic() and stuff like that when it's at
# the root.. the initial MapperEntity is more accurate for this case.
if len(path) == 1:
if not orm_util._entity_isa(query_entity.entity_zero, self.parent):
return
elif not orm_util._entity_isa(path[-1], self.parent):
return
subq = self._setup_query_from_rowproc(
context,
query_entity,
path,
path[-1],
loadopt,
adapter,
)
if subq is None:
return
assert subq.session is None
path = path[self.parent_property]
local_cols = self.parent_property.local_columns
# cache the loaded collections in the context
# so that inheriting mappers don't re-load when they
# call upon create_row_processor again
collections = path.get(context.attributes, "collections")
if collections is None:
collections = self._SubqCollections(context, subq)
path.set(context.attributes, "collections", collections)
if adapter:
local_cols = [adapter.columns[c] for c in local_cols]
if self.uselist:
self._create_collection_loader(
context, result, collections, local_cols, populators
)
else:
self._create_scalar_loader(
context, result, collections, local_cols, populators
)
def _create_collection_loader(
self, context, result, collections, local_cols, populators
):
tuple_getter = result._tuple_getter(local_cols)
def load_collection_from_subq(state, dict_, row):
collection = collections.get(tuple_getter(row), ())
state.get_impl(self.key).set_committed_value(
state, dict_, collection
)
def load_collection_from_subq_existing_row(state, dict_, row):
if self.key not in dict_:
load_collection_from_subq(state, dict_, row)
populators["new"].append((self.key, load_collection_from_subq))
populators["existing"].append(
(self.key, load_collection_from_subq_existing_row)
)
if context.invoke_all_eagers:
populators["eager"].append((self.key, collections.loader))
def _create_scalar_loader(
self, context, result, collections, local_cols, populators
):
tuple_getter = result._tuple_getter(local_cols)
def load_scalar_from_subq(state, dict_, row):
collection = collections.get(tuple_getter(row), (None,))
if len(collection) > 1:
util.warn(
"Multiple rows returned with "
"uselist=False for eagerly-loaded attribute '%s' " % self
)
scalar = collection[0]
state.get_impl(self.key).set_committed_value(state, dict_, scalar)
def load_scalar_from_subq_existing_row(state, dict_, row):
if self.key not in dict_:
load_scalar_from_subq(state, dict_, row)
populators["new"].append((self.key, load_scalar_from_subq))
populators["existing"].append(
(self.key, load_scalar_from_subq_existing_row)
)
if context.invoke_all_eagers:
populators["eager"].append((self.key, collections.loader))
@log.class_logger
@relationships.RelationshipProperty.strategy_for(lazy="joined")
@relationships.RelationshipProperty.strategy_for(lazy=False)
class JoinedLoader(AbstractRelationshipLoader):
"""Provide loading behavior for a :class:`.Relationship`
using joined eager loading.
"""
__slots__ = "join_depth"
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
self.join_depth = self.parent_property.join_depth
def init_class_attribute(self, mapper):
self.parent_property._get_strategy(
(("lazy", "select"),)
).init_class_attribute(mapper)
def setup_query(
self,
compile_state,
query_entity,
path,
loadopt,
adapter,
column_collection=None,
parentmapper=None,
chained_from_outerjoin=False,
**kwargs,
):
"""Add a left outer join to the statement that's being constructed."""
if not compile_state.compile_options._enable_eagerloads:
return
elif self.uselist:
compile_state.multi_row_eager_loaders = True
path = path[self.parent_property]
with_polymorphic = None
user_defined_adapter = (
self._init_user_defined_eager_proc(
loadopt, compile_state, compile_state.attributes
)
if loadopt
else False
)
if user_defined_adapter is not False:
# setup an adapter but dont create any JOIN, assume it's already
# in the query
(
clauses,
adapter,
add_to_collection,
) = self._setup_query_on_user_defined_adapter(
compile_state,
query_entity,
path,
adapter,
user_defined_adapter,
)
# don't do "wrap" for multi-row, we want to wrap
# limited/distinct SELECT,
# because we want to put the JOIN on the outside.
else:
# if not via query option, check for
# a cycle
if not path.contains(compile_state.attributes, "loader"):
if self.join_depth:
if path.length / 2 > self.join_depth:
return
elif path.contains_mapper(self.mapper):
return
# add the JOIN and create an adapter
(
clauses,
adapter,
add_to_collection,
chained_from_outerjoin,
) = self._generate_row_adapter(
compile_state,
query_entity,
path,
loadopt,
adapter,
column_collection,
parentmapper,
chained_from_outerjoin,
)
# for multi-row, we want to wrap limited/distinct SELECT,
# because we want to put the JOIN on the outside.
compile_state.eager_adding_joins = True
with_poly_entity = path.get(
compile_state.attributes, "path_with_polymorphic", None
)
if with_poly_entity is not None:
with_polymorphic = inspect(
with_poly_entity
).with_polymorphic_mappers
else:
with_polymorphic = None
path = path[self.entity]
loading._setup_entity_query(
compile_state,
self.mapper,
query_entity,
path,
clauses,
add_to_collection,
with_polymorphic=with_polymorphic,
parentmapper=self.mapper,
chained_from_outerjoin=chained_from_outerjoin,
)
has_nones = util.NONE_SET.intersection(compile_state.secondary_columns)
if has_nones:
if with_poly_entity is not None:
raise sa_exc.InvalidRequestError(
"Detected unaliased columns when generating joined "
"load. Make sure to use aliased=True or flat=True "
"when using joined loading with with_polymorphic()."
)
else:
compile_state.secondary_columns = [
c for c in compile_state.secondary_columns if c is not None
]
def _init_user_defined_eager_proc(
self, loadopt, compile_state, target_attributes
):
# check if the opt applies at all
if "eager_from_alias" not in loadopt.local_opts:
# nope
return False
path = loadopt.path.parent
# the option applies. check if the "user_defined_eager_row_processor"
# has been built up.
adapter = path.get(
compile_state.attributes, "user_defined_eager_row_processor", False
)
if adapter is not False:
# just return it
return adapter
# otherwise figure it out.
alias = loadopt.local_opts["eager_from_alias"]
root_mapper, prop = path[-2:]
if alias is not None:
if isinstance(alias, str):
alias = prop.target.alias(alias)
adapter = orm_util.ORMAdapter(
orm_util._TraceAdaptRole.JOINEDLOAD_USER_DEFINED_ALIAS,
prop.mapper,
selectable=alias,
equivalents=prop.mapper._equivalent_columns,
limit_on_entity=False,
)
else:
if path.contains(
compile_state.attributes, "path_with_polymorphic"
):
with_poly_entity = path.get(
compile_state.attributes, "path_with_polymorphic"
)
adapter = orm_util.ORMAdapter(
orm_util._TraceAdaptRole.JOINEDLOAD_PATH_WITH_POLYMORPHIC,
with_poly_entity,
equivalents=prop.mapper._equivalent_columns,
)
else:
adapter = compile_state._polymorphic_adapters.get(
prop.mapper, None
)
path.set(
target_attributes,
"user_defined_eager_row_processor",
adapter,
)
return adapter
def _setup_query_on_user_defined_adapter(
self, context, entity, path, adapter, user_defined_adapter
):
# apply some more wrapping to the "user defined adapter"
# if we are setting up the query for SQL render.
adapter = entity._get_entity_clauses(context)
if adapter and user_defined_adapter:
user_defined_adapter = user_defined_adapter.wrap(adapter)
path.set(
context.attributes,
"user_defined_eager_row_processor",
user_defined_adapter,
)
elif adapter:
user_defined_adapter = adapter
path.set(
context.attributes,
"user_defined_eager_row_processor",
user_defined_adapter,
)
add_to_collection = context.primary_columns
return user_defined_adapter, adapter, add_to_collection
def _generate_row_adapter(
self,
compile_state,
entity,
path,
loadopt,
adapter,
column_collection,
parentmapper,
chained_from_outerjoin,
):
with_poly_entity = path.get(
compile_state.attributes, "path_with_polymorphic", None
)
if with_poly_entity:
to_adapt = with_poly_entity
else:
insp = inspect(self.entity)
if insp.is_aliased_class:
alt_selectable = insp.selectable
else:
alt_selectable = None
to_adapt = orm_util.AliasedClass(
self.mapper,
alias=(
alt_selectable._anonymous_fromclause(flat=True)
if alt_selectable is not None
else None
),
flat=True,
use_mapper_path=True,
)
to_adapt_insp = inspect(to_adapt)
clauses = to_adapt_insp._memo(
("joinedloader_ormadapter", self),
orm_util.ORMAdapter,
orm_util._TraceAdaptRole.JOINEDLOAD_MEMOIZED_ADAPTER,
to_adapt_insp,
equivalents=self.mapper._equivalent_columns,
adapt_required=True,
allow_label_resolve=False,
anonymize_labels=True,
)
assert clauses.is_aliased_class
innerjoin = (
loadopt.local_opts.get("innerjoin", self.parent_property.innerjoin)
if loadopt is not None
else self.parent_property.innerjoin
)
if not innerjoin:
# if this is an outer join, all non-nested eager joins from
# this path must also be outer joins
chained_from_outerjoin = True
compile_state.create_eager_joins.append(
(
self._create_eager_join,
entity,
path,
adapter,
parentmapper,
clauses,
innerjoin,
chained_from_outerjoin,
loadopt._extra_criteria if loadopt else (),
)
)
add_to_collection = compile_state.secondary_columns
path.set(compile_state.attributes, "eager_row_processor", clauses)
return clauses, adapter, add_to_collection, chained_from_outerjoin
def _create_eager_join(
self,
compile_state,
query_entity,
path,
adapter,
parentmapper,
clauses,
innerjoin,
chained_from_outerjoin,
extra_criteria,
):
if parentmapper is None:
localparent = query_entity.mapper
else:
localparent = parentmapper
# whether or not the Query will wrap the selectable in a subquery,
# and then attach eager load joins to that (i.e., in the case of
# LIMIT/OFFSET etc.)
should_nest_selectable = (
compile_state.multi_row_eager_loaders
and compile_state._should_nest_selectable
)
query_entity_key = None
if (
query_entity not in compile_state.eager_joins
and not should_nest_selectable
and compile_state.from_clauses
):
indexes = sql_util.find_left_clause_that_matches_given(
compile_state.from_clauses, query_entity.selectable
)
if len(indexes) > 1:
# for the eager load case, I can't reproduce this right
# now. For query.join() I can.
raise sa_exc.InvalidRequestError(
"Can't identify which query entity in which to joined "
"eager load from. Please use an exact match when "
"specifying the join path."
)
if indexes:
clause = compile_state.from_clauses[indexes[0]]
# join to an existing FROM clause on the query.
# key it to its list index in the eager_joins dict.
# Query._compile_context will adapt as needed and
# append to the FROM clause of the select().
query_entity_key, default_towrap = indexes[0], clause
if query_entity_key is None:
query_entity_key, default_towrap = (
query_entity,
query_entity.selectable,
)
towrap = compile_state.eager_joins.setdefault(
query_entity_key, default_towrap
)
if adapter:
if getattr(adapter, "is_aliased_class", False):
# joining from an adapted entity. The adapted entity
# might be a "with_polymorphic", so resolve that to our
# specific mapper's entity before looking for our attribute
# name on it.
efm = adapter.aliased_insp._entity_for_mapper(
localparent
if localparent.isa(self.parent)
else self.parent
)
# look for our attribute on the adapted entity, else fall back
# to our straight property
onclause = getattr(efm.entity, self.key, self.parent_property)
else:
onclause = getattr(
orm_util.AliasedClass(
self.parent, adapter.selectable, use_mapper_path=True
),
self.key,
self.parent_property,
)
else:
onclause = self.parent_property
assert clauses.is_aliased_class
attach_on_outside = (
not chained_from_outerjoin
or not innerjoin
or innerjoin == "unnested"
or query_entity.entity_zero.represents_outer_join
)
extra_join_criteria = extra_criteria
additional_entity_criteria = compile_state.global_attributes.get(
("additional_entity_criteria", self.mapper), ()
)
if additional_entity_criteria:
extra_join_criteria += tuple(
ae._resolve_where_criteria(self.mapper)
for ae in additional_entity_criteria
if ae.propagate_to_loaders
)
if attach_on_outside:
# this is the "classic" eager join case.
eagerjoin = orm_util._ORMJoin(
towrap,
clauses.aliased_insp,
onclause,
isouter=not innerjoin
or query_entity.entity_zero.represents_outer_join
or (chained_from_outerjoin and isinstance(towrap, sql.Join)),
_left_memo=self.parent,
_right_memo=path[self.mapper],
_extra_criteria=extra_join_criteria,
)
else:
# all other cases are innerjoin=='nested' approach
eagerjoin = self._splice_nested_inner_join(
path, towrap, clauses, onclause, extra_join_criteria
)
compile_state.eager_joins[query_entity_key] = eagerjoin
# send a hint to the Query as to where it may "splice" this join
eagerjoin.stop_on = query_entity.selectable
if not parentmapper:
# for parentclause that is the non-eager end of the join,
# ensure all the parent cols in the primaryjoin are actually
# in the
# columns clause (i.e. are not deferred), so that aliasing applied
# by the Query propagates those columns outward.
# This has the effect
# of "undefering" those columns.
for col in sql_util._find_columns(
self.parent_property.primaryjoin
):
if localparent.persist_selectable.c.contains_column(col):
if adapter:
col = adapter.columns[col]
compile_state._append_dedupe_col_collection(
col, compile_state.primary_columns
)
if self.parent_property.order_by:
compile_state.eager_order_by += tuple(
(eagerjoin._target_adapter.copy_and_process)(
util.to_list(self.parent_property.order_by)
)
)
def _splice_nested_inner_join(
self,
path,
join_obj,
clauses,
onclause,
extra_criteria,
splicing=False,
detected_existing_path=None,
):
# recursive fn to splice a nested join into an existing one.
# splicing=False means this is the outermost call, and it
# should return a value. splicing=<from object> is the recursive
# form, where it can return None to indicate the end of the recursion
if splicing is False:
# first call is always handed a join object
# from the outside
assert isinstance(join_obj, orm_util._ORMJoin)
elif isinstance(join_obj, sql.selectable.FromGrouping):
return self._splice_nested_inner_join(
path,
join_obj.element,
clauses,
onclause,
extra_criteria,
splicing,
)
elif not isinstance(join_obj, orm_util._ORMJoin):
if path[-2].isa(splicing):
if detected_existing_path:
# TODO: refine this into a more efficient method
if not detected_existing_path.contains_mapper(splicing):
return None
elif path_registry.PathRegistry.coerce(
detected_existing_path[len(path) :]
).contains_mapper(splicing):
return None
return orm_util._ORMJoin(
join_obj,
clauses.aliased_insp,
onclause,
isouter=False,
_left_memo=splicing,
_right_memo=path[path[-1].mapper],
_extra_criteria=extra_criteria,
)
else:
return None
target_join = self._splice_nested_inner_join(
path,
join_obj.right,
clauses,
onclause,
extra_criteria,
# NOTE: this is the one place _right_memo is consumed
splicing=(
join_obj._right_memo[-1].mapper
if join_obj._right_memo is not None
else None
),
)
if target_join is None:
right_splice = False
target_join = self._splice_nested_inner_join(
path,
join_obj.left,
clauses,
onclause,
extra_criteria,
join_obj._left_memo,
detected_existing_path=join_obj._right_memo,
)
if target_join is None:
# should only return None when recursively called,
# e.g. splicing refers to a from obj
assert (
splicing is not False
), "assertion failed attempting to produce joined eager loads"
return None
else:
right_splice = True
if right_splice:
# for a right splice, attempt to flatten out
# a JOIN b JOIN c JOIN .. to avoid needless
# parenthesis nesting
if not join_obj.isouter and not target_join.isouter:
eagerjoin = join_obj._splice_into_center(target_join)
else:
eagerjoin = orm_util._ORMJoin(
join_obj.left,
target_join,
join_obj.onclause,
isouter=join_obj.isouter,
_left_memo=join_obj._left_memo,
)
else:
eagerjoin = orm_util._ORMJoin(
target_join,
join_obj.right,
join_obj.onclause,
isouter=join_obj.isouter,
_right_memo=join_obj._right_memo,
)
eagerjoin._target_adapter = target_join._target_adapter
return eagerjoin
def _create_eager_adapter(self, context, result, adapter, path, loadopt):
compile_state = context.compile_state
user_defined_adapter = (
self._init_user_defined_eager_proc(
loadopt, compile_state, context.attributes
)
if loadopt
else False
)
if user_defined_adapter is not False:
decorator = user_defined_adapter
# user defined eagerloads are part of the "primary"
# portion of the load.
# the adapters applied to the Query should be honored.
if compile_state.compound_eager_adapter and decorator:
decorator = decorator.wrap(
compile_state.compound_eager_adapter
)
elif compile_state.compound_eager_adapter:
decorator = compile_state.compound_eager_adapter
else:
decorator = path.get(
compile_state.attributes, "eager_row_processor"
)
if decorator is None:
return False
if self.mapper._result_has_identity_key(result, decorator):
return decorator
else:
# no identity key - don't return a row
# processor, will cause a degrade to lazy
return False
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
if not context.compile_state.compile_options._enable_eagerloads:
return
if not self.parent.class_manager[self.key].impl.supports_population:
raise sa_exc.InvalidRequestError(
"'%s' does not support object "
"population - eager loading cannot be applied." % self
)
if self.uselist:
context.loaders_require_uniquing = True
our_path = path[self.parent_property]
eager_adapter = self._create_eager_adapter(
context, result, adapter, our_path, loadopt
)
if eager_adapter is not False:
key = self.key
_instance = loading._instance_processor(
query_entity,
self.mapper,
context,
result,
our_path[self.entity],
eager_adapter,
)
if not self.uselist:
self._create_scalar_loader(context, key, _instance, populators)
else:
self._create_collection_loader(
context, key, _instance, populators
)
else:
self.parent_property._get_strategy(
(("lazy", "select"),)
).create_row_processor(
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
)
def _create_collection_loader(self, context, key, _instance, populators):
def load_collection_from_joined_new_row(state, dict_, row):
# note this must unconditionally clear out any existing collection.
# an existing collection would be present only in the case of
# populate_existing().
collection = attributes.init_state_collection(state, dict_, key)
result_list = util.UniqueAppender(
collection, "append_without_event"
)
context.attributes[(state, key)] = result_list
inst = _instance(row)
if inst is not None:
result_list.append(inst)
def load_collection_from_joined_existing_row(state, dict_, row):
if (state, key) in context.attributes:
result_list = context.attributes[(state, key)]
else:
# appender_key can be absent from context.attributes
# with isnew=False when self-referential eager loading
# is used; the same instance may be present in two
# distinct sets of result columns
collection = attributes.init_state_collection(
state, dict_, key
)
result_list = util.UniqueAppender(
collection, "append_without_event"
)
context.attributes[(state, key)] = result_list
inst = _instance(row)
if inst is not None:
result_list.append(inst)
def load_collection_from_joined_exec(state, dict_, row):
_instance(row)
populators["new"].append(
(self.key, load_collection_from_joined_new_row)
)
populators["existing"].append(
(self.key, load_collection_from_joined_existing_row)
)
if context.invoke_all_eagers:
populators["eager"].append(
(self.key, load_collection_from_joined_exec)
)
def _create_scalar_loader(self, context, key, _instance, populators):
def load_scalar_from_joined_new_row(state, dict_, row):
# set a scalar object instance directly on the parent
# object, bypassing InstrumentedAttribute event handlers.
dict_[key] = _instance(row)
def load_scalar_from_joined_existing_row(state, dict_, row):
# call _instance on the row, even though the object has
# been created, so that we further descend into properties
existing = _instance(row)
# conflicting value already loaded, this shouldn't happen
if key in dict_:
if existing is not dict_[key]:
util.warn(
"Multiple rows returned with "
"uselist=False for eagerly-loaded attribute '%s' "
% self
)
else:
# this case is when one row has multiple loads of the
# same entity (e.g. via aliasing), one has an attribute
# that the other doesn't.
dict_[key] = existing
def load_scalar_from_joined_exec(state, dict_, row):
_instance(row)
populators["new"].append((self.key, load_scalar_from_joined_new_row))
populators["existing"].append(
(self.key, load_scalar_from_joined_existing_row)
)
if context.invoke_all_eagers:
populators["eager"].append(
(self.key, load_scalar_from_joined_exec)
)
@log.class_logger
@relationships.RelationshipProperty.strategy_for(lazy="selectin")
class SelectInLoader(PostLoader, util.MemoizedSlots):
__slots__ = (
"join_depth",
"omit_join",
"_parent_alias",
"_query_info",
"_fallback_query_info",
)
query_info = collections.namedtuple(
"queryinfo",
[
"load_only_child",
"load_with_join",
"in_expr",
"pk_cols",
"zero_idx",
"child_lookup_cols",
],
)
_chunksize = 500
def __init__(self, parent, strategy_key):
super().__init__(parent, strategy_key)
self.join_depth = self.parent_property.join_depth
is_m2o = self.parent_property.direction is interfaces.MANYTOONE
if self.parent_property.omit_join is not None:
self.omit_join = self.parent_property.omit_join
else:
lazyloader = self.parent_property._get_strategy(
(("lazy", "select"),)
)
if is_m2o:
self.omit_join = lazyloader.use_get
else:
self.omit_join = self.parent._get_clause[0].compare(
lazyloader._rev_lazywhere,
use_proxies=True,
compare_keys=False,
equivalents=self.parent._equivalent_columns,
)
if self.omit_join:
if is_m2o:
self._query_info = self._init_for_omit_join_m2o()
self._fallback_query_info = self._init_for_join()
else:
self._query_info = self._init_for_omit_join()
else:
self._query_info = self._init_for_join()
def _init_for_omit_join(self):
pk_to_fk = dict(
self.parent_property._join_condition.local_remote_pairs
)
pk_to_fk.update(
(equiv, pk_to_fk[k])
for k in list(pk_to_fk)
for equiv in self.parent._equivalent_columns.get(k, ())
)
pk_cols = fk_cols = [
pk_to_fk[col] for col in self.parent.primary_key if col in pk_to_fk
]
if len(fk_cols) > 1:
in_expr = sql.tuple_(*fk_cols)
zero_idx = False
else:
in_expr = fk_cols[0]
zero_idx = True
return self.query_info(False, False, in_expr, pk_cols, zero_idx, None)
def _init_for_omit_join_m2o(self):
pk_cols = self.mapper.primary_key
if len(pk_cols) > 1:
in_expr = sql.tuple_(*pk_cols)
zero_idx = False
else:
in_expr = pk_cols[0]
zero_idx = True
lazyloader = self.parent_property._get_strategy((("lazy", "select"),))
lookup_cols = [lazyloader._equated_columns[pk] for pk in pk_cols]
return self.query_info(
True, False, in_expr, pk_cols, zero_idx, lookup_cols
)
def _init_for_join(self):
self._parent_alias = AliasedClass(self.parent.class_)
pa_insp = inspect(self._parent_alias)
pk_cols = [
pa_insp._adapt_element(col) for col in self.parent.primary_key
]
if len(pk_cols) > 1:
in_expr = sql.tuple_(*pk_cols)
zero_idx = False
else:
in_expr = pk_cols[0]
zero_idx = True
return self.query_info(False, True, in_expr, pk_cols, zero_idx, None)
def init_class_attribute(self, mapper):
self.parent_property._get_strategy(
(("lazy", "select"),)
).init_class_attribute(mapper)
def create_row_processor(
self,
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
):
if context.refresh_state:
return self._immediateload_create_row_processor(
context,
query_entity,
path,
loadopt,
mapper,
result,
adapter,
populators,
)
(
effective_path,
run_loader,
execution_options,
recursion_depth,
) = self._setup_for_recursion(
context, path, loadopt, join_depth=self.join_depth
)
if not run_loader:
return
if not context.compile_state.compile_options._enable_eagerloads:
return
if not self.parent.class_manager[self.key].impl.supports_population:
raise sa_exc.InvalidRequestError(
"'%s' does not support object "
"population - eager loading cannot be applied." % self
)
# a little dance here as the "path" is still something that only
# semi-tracks the exact series of things we are loading, still not
# telling us about with_polymorphic() and stuff like that when it's at
# the root.. the initial MapperEntity is more accurate for this case.
if len(path) == 1:
if not orm_util._entity_isa(query_entity.entity_zero, self.parent):
return
elif not orm_util._entity_isa(path[-1], self.parent):
return
selectin_path = effective_path
path_w_prop = path[self.parent_property]
# build up a path indicating the path from the leftmost
# entity to the thing we're subquery loading.
with_poly_entity = path_w_prop.get(
context.attributes, "path_with_polymorphic", None
)
if with_poly_entity is not None:
effective_entity = inspect(with_poly_entity)
else:
effective_entity = self.entity
loading.PostLoad.callable_for_path(
context,
selectin_path,
self.parent,
self.parent_property,
self._load_for_path,
effective_entity,
loadopt,
recursion_depth,
execution_options,
)
def _load_for_path(
self,
context,
path,
states,
load_only,
effective_entity,
loadopt,
recursion_depth,
execution_options,
):
if load_only and self.key not in load_only:
return
query_info = self._query_info
if query_info.load_only_child:
our_states = collections.defaultdict(list)
none_states = []
mapper = self.parent
for state, overwrite in states:
state_dict = state.dict
related_ident = tuple(
mapper._get_state_attr_by_column(
state,
state_dict,
lk,
passive=attributes.PASSIVE_NO_FETCH,
)
for lk in query_info.child_lookup_cols
)
# if the loaded parent objects do not have the foreign key
# to the related item loaded, then degrade into the joined
# version of selectinload
if LoaderCallableStatus.PASSIVE_NO_RESULT in related_ident:
query_info = self._fallback_query_info
break
# organize states into lists keyed to particular foreign
# key values.
if None not in related_ident:
our_states[related_ident].append(
(state, state_dict, overwrite)
)
else:
# For FK values that have None, add them to a
# separate collection that will be populated separately
none_states.append((state, state_dict, overwrite))
# note the above conditional may have changed query_info
if not query_info.load_only_child:
our_states = [
(state.key[1], state, state.dict, overwrite)
for state, overwrite in states
]
pk_cols = query_info.pk_cols
in_expr = query_info.in_expr
if not query_info.load_with_join:
# in "omit join" mode, the primary key column and the
# "in" expression are in terms of the related entity. So
# if the related entity is polymorphic or otherwise aliased,
# we need to adapt our "pk_cols" and "in_expr" to that
# entity. in non-"omit join" mode, these are against the
# parent entity and do not need adaption.
if effective_entity.is_aliased_class:
pk_cols = [
effective_entity._adapt_element(col) for col in pk_cols
]
in_expr = effective_entity._adapt_element(in_expr)
bundle_ent = orm_util.Bundle("pk", *pk_cols)
bundle_sql = bundle_ent.__clause_element__()
entity_sql = effective_entity.__clause_element__()
q = Select._create_raw_select(
_raw_columns=[bundle_sql, entity_sql],
_label_style=LABEL_STYLE_TABLENAME_PLUS_COL,
_compile_options=ORMCompileState.default_compile_options,
_propagate_attrs={
"compile_state_plugin": "orm",
"plugin_subject": effective_entity,
},
)
if not query_info.load_with_join:
# the Bundle we have in the "omit_join" case is against raw, non
# annotated columns, so to ensure the Query knows its primary
# entity, we add it explicitly. If we made the Bundle against
# annotated columns, we hit a performance issue in this specific
# case, which is detailed in issue #4347.
q = q.select_from(effective_entity)
else:
# in the non-omit_join case, the Bundle is against the annotated/
# mapped column of the parent entity, but the #4347 issue does not
# occur in this case.
q = q.select_from(self._parent_alias).join(
getattr(self._parent_alias, self.parent_property.key).of_type(
effective_entity
)
)
q = q.filter(in_expr.in_(sql.bindparam("primary_keys")))
# a test which exercises what these comments talk about is
# test_selectin_relations.py -> test_twolevel_selectin_w_polymorphic
#
# effective_entity above is given to us in terms of the cached
# statement, namely this one:
orig_query = context.compile_state.select_statement
# the actual statement that was requested is this one:
# context_query = context.query
#
# that's not the cached one, however. So while it is of the identical
# structure, if it has entities like AliasedInsp, which we get from
# aliased() or with_polymorphic(), the AliasedInsp will likely be a
# different object identity each time, and will not match up
# hashing-wise to the corresponding AliasedInsp that's in the
# cached query, meaning it won't match on paths and loader lookups
# and loaders like this one will be skipped if it is used in options.
#
# as it turns out, standard loader options like selectinload(),
# lazyload() that have a path need
# to come from the cached query so that the AliasedInsp etc. objects
# that are in the query line up with the object that's in the path
# of the strategy object. however other options like
# with_loader_criteria() that doesn't have a path (has a fixed entity)
# and needs to have access to the latest closure state in order to
# be correct, we need to use the uncached one.
#
# as of #8399 we let the loader option itself figure out what it
# wants to do given cached and uncached version of itself.
effective_path = path[self.parent_property]
if orig_query is context.query:
new_options = orig_query._with_options
else:
cached_options = orig_query._with_options
uncached_options = context.query._with_options
# propagate compile state options from the original query,
# updating their "extra_criteria" as necessary.
# note this will create a different cache key than
# "orig" options if extra_criteria is present, because the copy
# of extra_criteria will have different boundparam than that of
# the QueryableAttribute in the path
new_options = [
orig_opt._adapt_cached_option_to_uncached_option(
context, uncached_opt
)
for orig_opt, uncached_opt in zip(
cached_options, uncached_options
)
]
if loadopt and loadopt._extra_criteria:
new_options += (
orm_util.LoaderCriteriaOption(
effective_entity,
loadopt._generate_extra_criteria(context),
),
)
if recursion_depth is not None:
effective_path = effective_path._truncate_recursive()
q = q.options(*new_options)
q = q._update_compile_options({"_current_path": effective_path})
if context.populate_existing:
q = q.execution_options(populate_existing=True)
if self.parent_property.order_by:
if not query_info.load_with_join:
eager_order_by = self.parent_property.order_by
if effective_entity.is_aliased_class:
eager_order_by = [
effective_entity._adapt_element(elem)
for elem in eager_order_by
]
q = q.order_by(*eager_order_by)
else:
def _setup_outermost_orderby(compile_context):
compile_context.eager_order_by += tuple(
util.to_list(self.parent_property.order_by)
)
q = q._add_context_option(
_setup_outermost_orderby, self.parent_property
)
if query_info.load_only_child:
self._load_via_child(
our_states,
none_states,
query_info,
q,
context,
execution_options,
)
else:
self._load_via_parent(
our_states, query_info, q, context, execution_options
)
def _load_via_child(
self,
our_states,
none_states,
query_info,
q,
context,
execution_options,
):
uselist = self.uselist
# this sort is really for the benefit of the unit tests
our_keys = sorted(our_states)
while our_keys:
chunk = our_keys[0 : self._chunksize]
our_keys = our_keys[self._chunksize :]
data = {
k: v
for k, v in context.session.execute(
q,
params={
"primary_keys": [
key[0] if query_info.zero_idx else key
for key in chunk
]
},
execution_options=execution_options,
).unique()
}
for key in chunk:
# for a real foreign key and no concurrent changes to the
# DB while running this method, "key" is always present in
# data. However, for primaryjoins without real foreign keys
# a non-None primaryjoin condition may still refer to no
# related object.
related_obj = data.get(key, None)
for state, dict_, overwrite in our_states[key]:
if not overwrite and self.key in dict_:
continue
state.get_impl(self.key).set_committed_value(
state,
dict_,
related_obj if not uselist else [related_obj],
)
# populate none states with empty value / collection
for state, dict_, overwrite in none_states:
if not overwrite and self.key in dict_:
continue
# note it's OK if this is a uselist=True attribute, the empty
# collection will be populated
state.get_impl(self.key).set_committed_value(state, dict_, None)
def _load_via_parent(
self, our_states, query_info, q, context, execution_options
):
uselist = self.uselist
_empty_result = () if uselist else None
while our_states:
chunk = our_states[0 : self._chunksize]
our_states = our_states[self._chunksize :]
primary_keys = [
key[0] if query_info.zero_idx else key
for key, state, state_dict, overwrite in chunk
]
data = collections.defaultdict(list)
for k, v in itertools.groupby(
context.session.execute(
q,
params={"primary_keys": primary_keys},
execution_options=execution_options,
).unique(),
lambda x: x[0],
):
data[k].extend(vv[1] for vv in v)
for key, state, state_dict, overwrite in chunk:
if not overwrite and self.key in state_dict:
continue
collection = data.get(key, _empty_result)
if not uselist and collection:
if len(collection) > 1:
util.warn(
"Multiple rows returned with "
"uselist=False for eagerly-loaded "
"attribute '%s' " % self
)
state.get_impl(self.key).set_committed_value(
state, state_dict, collection[0]
)
else:
# note that empty tuple set on uselist=False sets the
# value to None
state.get_impl(self.key).set_committed_value(
state, state_dict, collection
)
def single_parent_validator(desc, prop):
def _do_check(state, value, oldvalue, initiator):
if value is not None and initiator.key == prop.key:
hasparent = initiator.hasparent(attributes.instance_state(value))
if hasparent and oldvalue is not value:
raise sa_exc.InvalidRequestError(
"Instance %s is already associated with an instance "
"of %s via its %s attribute, and is only allowed a "
"single parent."
% (orm_util.instance_str(value), state.class_, prop),
code="bbf1",
)
return value
def append(state, value, initiator):
return _do_check(state, value, None, initiator)
def set_(state, value, oldvalue, initiator):
return _do_check(state, value, oldvalue, initiator)
event.listen(
desc, "append", append, raw=True, retval=True, active_history=True
)
event.listen(desc, "set", set_, raw=True, retval=True, active_history=True)