o
    i e+                     @   s  d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dd ZG dd dej Z!			dddZ"e#e!e" dS )zKA Variable class that is replicated to logical cores for model parallelism.    )absolute_import)division)print_function)abcN)xla_sharding)tpu_util)context)config)ops)tensor_conversion_registry)control_flow_ops)gen_resource_variable_ops)gen_tpu_partition_ops)variable_scope	variables)save_context)basec                 K   sB   t |j | ||fi |W  d    S 1 sw   Y  d S N)r
   device)	update_fnvarvaluekwargs r   c/var/www/myenv/lib/python3.10/site-packages/tensorflow/python/distribute/tpu_replicated_variable.py_on_device_update&   s   $r   c                   @   sR  e Zd ZdZdAddZdd Zedd Zedd	 Zed
d Z	edd Z
edd Zejdd Zejdd Zedd Zedd Zedd Zedd Zedd Zedd Zed d! Zed"d# Zd$d% Zd&d' Zed(d) Zed*d+ Zd,d- ZdBd0d1Zd2d3 Zd4d5 ZdCd7d8Z dCd9d:Z!dCd;d<Z"d=d> Z#d?d@ Z$d.S )DTPUReplicatedVariablea  Container for replicated `Variables` that are treated as a single variable.

  This class maintains a list of replicated variables that are stored on
  separate logic TPU devices. TF2XLA bridge accesses these variables as
  if they were a single variable.
  c                    s   t  tjr rtdd  D rtd  t fdd D r/tddd  D  t fdd D rFtd	d
d  D   | _|| _| jdd | _	d| _
dS )a  Treats `variables` as a replicated list of `tf.Variable`s.

    Example:

    ```
    variables = [
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
    ]
    replicated_variable = TPUReplicatedVariable(variables)
    assert replicated_variable.shape.as_list() == [10, 100]
    ```

    Args:
      variables: A list of `ResourceVariable`s that comprise this replicated
        variable. Variables should not be shared between different
        `TPUReplicatedVariable` objects.
      name: String. Name of this container. Defaults to "TPUReplicatedVariable".
    c                 s   s    | ]
}t |tj V  qd S r   )
isinstancevariables_libVariable.0vr   r   r   	<genexpr>I   s    
z1TPUReplicatedVariable.__init__.<locals>.<genexpr>zSArgument `variables` should be a non-empty list of `variables.Variable`s. Received c                 3        | ]}|j  d  j kV  qdS r   Ndtyper!   r   r   r   r$   N       zPAll elements in argument `variables` must have the same dtype. Received dtypes: c                 S      g | ]}|j qS r   r'   r!   r   r   r   
<listcomp>Q       z2TPUReplicatedVariable.__init__.<locals>.<listcomp>c                 3   r%   r&   shaper!   r   r   r   r$   S   r)   zPAll elements in argument `variables` must have the same shape. Received shapes: c                 S   r*   r   r-   r!   r   r   r   r+   V   r,   :r   N)r   r   Sequenceany	TypeError
ValueError_vars_namesplit_common_name_cached_value)selfr   namer   r   r   __init__3   s,   
zTPUReplicatedVariable.__init__c                 C   s
   t | jS )zBReturn an iterable for accessing the underlying sharded variables.)iterr4   r9   r   r   r   __iter__]   s   
zTPUReplicatedVariable.__iter__c                 C      | j S )z0The name of this object. Used for checkpointing.)r5   r=   r   r   r   r:   a   s   zTPUReplicatedVariable.namec                 C      | j d jS )z,The dtype of all `Variable`s in this object.r   )r4   r(   r=   r   r   r   r(   f      zTPUReplicatedVariable.dtypec                 C   r@   Nr   )r4   is_initializedr=   r   r   r   rC   k      z$TPUReplicatedVariable.is_initializedc                 C   r@   rB   )r4   	trainabler=   r   r   r   rE   o   rD   zTPUReplicatedVariable.trainablec                 C   r@   )zThe device this variable is on.r   )r4   r   r=   r   r   r   r   s   rA   zTPUReplicatedVariable.devicec                 c   s:    | j j  d V  W d    d S 1 sw   Y  d S r   )handlegraph
as_defaultr=   r   r   r   _handle_graphx   s   "z#TPUReplicatedVariable._handle_graphc                 c   sP    | j d ur#t| j g d V  W d    d S 1 sw   Y  d S d V  d S r   )r8   r
   control_dependenciesr=   r   r   r   _assign_dependencies}   s   
"
z*TPUReplicatedVariable._assign_dependenciesc                 C   r@   rB   )r4   
constraintr=   r   r   r   rL      rD   z TPUReplicatedVariable.constraintc                 C   r@   rB   )r4   _in_graph_moder=   r   r   r   rM      rD   z$TPUReplicatedVariable._in_graph_modec                 C   r@   rB   )r4   
_unique_idr=   r   r   r   rN      rD   z TPUReplicatedVariable._unique_idc                 C   r@   rB   )r4   rG   r=   r   r   r   rG      rD   zTPUReplicatedVariable.graphc                 C   r?   r   )r7   r=   r   r   r   _shared_name   s   z"TPUReplicatedVariable._shared_namec                 C      t jjS r   )r   VariableSynchronizationNONEr=   r   r   r   synchronization      z%TPUReplicatedVariable.synchronizationc                 C   rP   r   )r   VariableAggregationrR   r=   r   r   r   aggregation   rT   z!TPUReplicatedVariable.aggregationc                 C   s   t  r
| jd gS | jS )zThe list of `Variables`.r   )r   in_save_contextr4   r=   r   r   r   r      s   zTPUReplicatedVariable.variablesc                 K   s   | j d }|j|||fi |}| j dd D ]}|| ||< ||j ||j< ||j q|| || < ||j || < ||  |S )zFor implementing `Trackable`.r      N)r4   _export_to_saved_model_graphrF   append)r9   
object_map
tensor_mapoptionsr   	first_varresource_listr#   r   r   r   rY      s   

z2TPUReplicatedVariable._export_to_saved_model_graphc                 C   s   t j| jd iS rB   )	trackableVARIABLE_VALUE_KEYr4   r=   r   r   r   !_gather_saveables_for_saved_model      z7TPUReplicatedVariable._gather_saveables_for_saved_modelc                 C   r@   rB   )r4   r.   r=   r   r   r   r.      rD   zTPUReplicatedVariable.shapec                 C   s   t  st r| jd jS t d u rtdt	 / t
| dd }|d u s+t r:tjdd | jD g dd}n
tj|jgg dd}W d    n1 sNw   Y  t|S )	Nr   zQTPUReplicatedVariable.handle is not available outside tpu context or save context_packed_varc                 S   r*   r   )rF   r!   r   r   r   r+      r,   z0TPUReplicatedVariable.handle.<locals>.<listcomp>F)partition_dims	is_packedT)r   rW   r   executing_eagerlyr4   rF   r   enclosing_tpu_contextNotImplementedErroroutside_or_skip_tpu_contextgetattrr	   get_soft_device_placementtpu_partition_opstpu_partitioned_input_v2packed_handler   	replicate)r9   
packed_vartensorr   r   r   rF      s"   


zTPUReplicatedVariable.handlec                 C   s   t | j| jS r   )r   read_variable_oprF   r(   r=   r   r   r   _read_variable_op   rc   z'TPUReplicatedVariable._read_variable_opNFc                 C   s   t  du r
|  S |  S )z Converts a variable to a tensor.N)r   rh   
read_valuert   )r9   r(   r:   as_refr   r   r   _dense_var_to_tensor   s   z*TPUReplicatedVariable._dense_var_to_tensorc                 C   s   | j d  S rB   )r4   ru   r=   r   r   r   ru      s   z TPUReplicatedVariable.read_valuec                    s4   t j|d| jd tjt fdd| jD  S )z;Converts the value to tensor and updates the variable list.value_in_tensor)r:   r(   c                 3   s$    | ]}t | fi V  qd S r   )r   r!   input_tensorr   r   r   r   r$      s
    
z0TPUReplicatedVariable._update.<locals>.<genexpr>)r
   convert_to_tensorr(   r   grouptupler   )r9   r   r   r   r   ry   r   _update   s   zTPUReplicatedVariable._updateTc                 C   J   t  d u s
t rdd }| j|||||dS t tj| ||||dS )Nc                 _      | j |i |S r   )assignr   akar   r   r   <lambda>       z.TPUReplicatedVariable.assign.<locals>.<lambda>r   use_lockingr:   ru   )r   rh   r   rg   r~   make_raw_assign_fnr   assign_variable_op)r9   r   r   r:   ru   	assign_fnr   r   r   r      $   zTPUReplicatedVariable.assignc                 C   r   )Nc                 _   r   r   )
assign_subr   r   r   r   r     r   z2TPUReplicatedVariable.assign_sub.<locals>.<lambda>r   )r   rh   r   rg   r~   r   r   assign_sub_variable_op)r9   r   r   r:   ru   assign_sub_fnr   r   r   r      r   z TPUReplicatedVariable.assign_subc                 C   r   )Nc                 _   r   r   )
assign_addr   r   r   r   r     r   z2TPUReplicatedVariable.assign_add.<locals>.<lambda>r   )r   rh   r   rg   r~   r   r   assign_add_variable_op)r9   r   r   r:   ru   assign_add_fnr   r   r   r     r   z TPUReplicatedVariable.assign_addc                 C   *   d dd t| jD }d| jj|f S )N,
c                 s        | ]\}}d ||f V  qdS )z  %d: %sNr   r"   ir#   r   r   r   r$   $      
z0TPUReplicatedVariable.__str__.<locals>.<genexpr>	%s:{
%s
}join	enumerater4   	__class____name__)r9   	debug_strr   r   r   __str__#     
zTPUReplicatedVariable.__str__c                 C   r   )Nr   c                 s   r   )z  %d: %rNr   r   r   r   r   r$   )  r   z1TPUReplicatedVariable.__repr__.<locals>.<genexpr>r   r   )r9   
debug_reprr   r   r   __repr__(  r   zTPUReplicatedVariable.__repr__)r   NNF)FNT)%r   
__module____qualname____doc__r;   r>   propertyr:   r(   rC   rE   r   
contextlibcontextmanagerrI   rK   rL   rM   rN   rG   rO   rS   rV   r   rY   rb   r.   rF   rt   rw   ru   r~   r   r   r   r   r   r   r   r   r   r   +   sb    
*





















r   Fc                 C   s   | j |||dS )N)r(   r:   rv   )rw   )r   r(   r:   rv   r   r   r   %_tensor_conversion_tpu_replicated_var0  s   r   r   )$r   
__future__r   r   r   collectionsr   r   +tensorflow.python.compiler.xla.experimentalr   tensorflow.python.distributer   tensorflow.python.eagerr   tensorflow.python.frameworkr	   r
   r   tensorflow.python.opsr   r   r   rm   r   r   r   tensorflow.python.saved_modelr   tensorflow.python.trackabler   r`   r   r    r   r   #register_tensor_conversion_functionr   r   r   r   <module>   s<     
