o
    i e5                     @   s   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 G dd	 d	e
jZG d
d deZ			dddZeee dS )zFA variable which packs a list of variables distributed across devices.    )device_util)context)ops)tensor_conversion_registry)math_ops)resource_variable_opsc                       s  e Zd ZdZd3 fdd	Zedd Zdd Zd	d
 Zdd Z	dd Z
edd Zedd Z fddZdd Zd4 fdd	Z fddZd5ddZd5ddZd5d d!Zd6d#d$Zd6d%d&Zd6d'd(Zd6d)d*Zd6d+d,Zd6d-d.Zd6d/d0Zd7 fd1d2	Z  ZS )8PackedDistributedVariablea$  A variable which packs multiple variables distributed across devices.

  It's only supported when eager execution is enabled.
  For op-by-op execution, use an unpacked handle on the current device; for
  function execution, use the packed handle to reduce the overhead of function
  calls.
  Nc           	   	      s  t  std|stdt|D ]\}}t|s%td|t|f q|| _dd |D | _t 	  t j
|ddds}t d	d |D }t |}d
|t  f }tt| jd i d|d jd|d jd|d jd|d|d jd|d jd|d jd|d jd|d|d|dddddddddddddd W d   n1 sw   Y  W d   dS W d   dS 1 sw   Y  dS )!a  Packs a list of variables which are distributed across devices.

    Args:
      distributed_variables: A list of distributed Variables to pack.
      name: Optional name for the variable. Defaults to `'Variable'` and gets
        uniquified automatically.
    z:PackedDistributedVariable should be created in eager mode.z-Expect a non-empty list of variables to pack.zHExpect a list of ResourceVariables to pack, but the %d-th variable is %sc                 S      g | ]}|j qS  )device).0vr
   r
   g/var/www/myenv/lib/python3.10/site-packages/tensorflow/python/distribute/packed_distributed_variable.py
<listcomp>5       z6PackedDistributedVariable.__init__.<locals>.<listcomp>VariableF)skip_on_eagerc                 S   r	   r
   )handle)r   varr
   r
   r   r   9   r   z%s_%d	trainabler   shapedtyper   synchronization
constraintaggregationdistribute_strategyname	unique_idhandle_namegraph_elementNinitial_valueinitializer_opis_initialized_opcached_valuecaching_deviceis_distributed_variablesTr
   )r   #executing_eagerly_outside_functions
ValueError	enumerater   is_resource_variabletype_distributed_variables_devices
init_scope
name_scopepack_eager_tensorsname_from_scope_nameuidsuperr   __init__r   r   r   r   r   r   _distribute_strategy)	selfdistributed_variablesr   unused_kwargsir   r   r   r   	__class__r
   r   r3   "   s|   










	
"z"PackedDistributedVariable.__init__c                 C      | j S N)r,   r5   r
   r
   r   devicesP      z!PackedDistributedVariable.devicesc                 C   s
   t | |S r<   )PackedVarAndDevicer5   r   r
   r
   r   	on_deviceT      
z#PackedDistributedVariable.on_devicec                 C   s6   t | jD ]\}}||kr| j|   S qtd| )NzDevice %s is not found)r(   r,   r+   r'   )r5   r   r8   dr
   r
   r   get_var_on_deviceW   s
   z+PackedDistributedVariable.get_var_on_devicec                 C   s   t t  }| |S r<   )r   canonicalizecurrentrE   )r5   current_devicer
   r
   r   get_var_on_current_device]   s   
z3PackedDistributedVariable.get_var_on_current_devicec                 C   s   |  |jS )z>Returns the Tensor used as the initial value for the variable.)rE   r    rA   r
   r
   r   r    a   s   z'PackedDistributedVariable.initial_valuec                 C   s   t  r	|  jS | jS r<   )r   executing_eagerlyrI   r   _handler=   r
   r
   r   r   e   s   
z PackedDistributedVariable.handlec                 C   r;   r<   )rK   r=   r
   r
   r   packed_handlel   r?   z'PackedDistributedVariable.packed_handlec                    s"   t  r
|   S tt|  S r<   )r   rJ   rI   valuer2   r   _read_variable_opr=   r9   r
   r   rN   p   s   z+PackedDistributedVariable._read_variable_opc                 C   s   |   S r<   )rN   r=   r
   r
   r   rM   v   s   zPackedDistributedVariable.valuec              	      sF  t  r,| jd  }| jdd D ]
}t|| }qtj|| jd  |d}|S t| jd  t	t
| |}W d    n1 sGw   Y  | jdd D ]%}t| t	t
| |}W d    n1 smw   Y  t||}qSt| jd  t	t
| |}W d    n1 sw   Y  tj|||d}|S )Nr      )r   )r   rJ   r+   is_initializedr   logical_andr   r   r,   r2   r   )r5   r   resultr   rD   initializedr9   r
   r   rQ   y   s:   z(PackedDistributedVariable.is_initializedc                    s6   t  r||  |fi |S |tt| |fi |S r<   )r   rJ   rI   r2   r   )r5   	update_fnrM   kwargsr9   r
   r   _update   s   z!PackedDistributedVariable._updateTc                 C      dd }| j |||||dS )Nc                 _      | j |i |S r<   )
assign_subr   akwr
   r
   r   <lambda>       z6PackedDistributedVariable.assign_sub.<locals>.<lambda>rU   rM   use_lockingr   
read_valuerW   )r5   deltara   r   rb   assign_sub_fnr
   r
   r   rZ         z$PackedDistributedVariable.assign_subc                 C   rX   )Nc                 _   rY   r<   )
assign_addr[   r
   r
   r   r^      r_   z6PackedDistributedVariable.assign_add.<locals>.<lambda>r`   rc   )r5   rd   ra   r   rb   assign_add_fnr
   r
   r   rg      rf   z$PackedDistributedVariable.assign_addc                 C   rX   )Nc                 _   rY   r<   )assignr[   r
   r
   r   r^      r_   z2PackedDistributedVariable.assign.<locals>.<lambda>r`   rc   )r5   rM   ra   r   rb   	assign_fnr
   r
   r   ri      rf   z PackedDistributedVariable.assignFc                 C      dd }| j ||||dS )Nc                 _   rY   r<   )scatter_subr[   r
   r
   r   r^      r_   z7PackedDistributedVariable.scatter_sub.<locals>.<lambda>rU   rM   ra   r   rc   )r5   sparse_deltara   r   scatter_sub_fnr
   r
   r   rl         z%PackedDistributedVariable.scatter_subc                 C   rk   )Nc                 _   rY   r<   )scatter_addr[   r
   r
   r   r^      r_   z7PackedDistributedVariable.scatter_add.<locals>.<lambda>rm   rc   )r5   rn   ra   r   scatter_add_fnr
   r
   r   rq      rp   z%PackedDistributedVariable.scatter_addc                 C   rk   )Nc                 _   rY   r<   )scatter_mulr[   r
   r
   r   r^      r_   z7PackedDistributedVariable.scatter_mul.<locals>.<lambda>rm   rc   )r5   rn   ra   r   scatter_mul_fnr
   r
   r   rs      rp   z%PackedDistributedVariable.scatter_mulc                 C   rk   )Nc                 _   rY   r<   )scatter_divr[   r
   r
   r   r^      r_   z7PackedDistributedVariable.scatter_div.<locals>.<lambda>rm   rc   )r5   rn   ra   r   scatter_div_fnr
   r
   r   ru      rp   z%PackedDistributedVariable.scatter_divc                 C   rk   )Nc                 _   rY   r<   )scatter_minr[   r
   r
   r   r^      r_   z7PackedDistributedVariable.scatter_min.<locals>.<lambda>rm   rc   )r5   rn   ra   r   scatter_min_fnr
   r
   r   rw      rp   z%PackedDistributedVariable.scatter_minc                 C   rk   )Nc                 _   rY   r<   )scatter_maxr[   r
   r
   r   r^      r_   z7PackedDistributedVariable.scatter_max.<locals>.<lambda>rm   rc   )r5   rn   ra   r   scatter_max_fnr
   r
   r   ry      rp   z%PackedDistributedVariable.scatter_maxc                 C   rk   )Nc                 _   rY   r<   )scatter_updater[   r
   r
   r   r^      r_   z:PackedDistributedVariable.scatter_update.<locals>.<lambda>rm   rc   )r5   rn   ra   r   scatter_update_fnr
   r
   r   r{      rp   z(PackedDistributedVariable.scatter_updatec                    s2   t  r|  j|||dS tt| j|||dS N)r   r   as_ref)r   rJ   rI   _dense_var_to_tensorr2   r   r5   r   r   r~   r9   r
   r   r      s   
z.PackedDistributedVariable._dense_var_to_tensor)NNr<   NNTFNNNF)__name__
__module____qualname____doc__r3   propertyr>   rB   rE   rI   r    r   rL   rN   rM   rQ   rW   rZ   rg   ri   rl   rq   rs   ru   rw   ry   r{   r   __classcell__r
   r
   r9   r   r      s8    .




	
	
	





r   c                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	dd Z
dd Ze	dd Ze	dd Zdd Ze	dd Zd3ddZd3ddZd3ddZd4d!d"Zd4d#d$Zd4d%d&Zd4d'd(Zd4d)d*Zd4d+d,Zd4d-d.Zd5d/d0Zd1d2 ZdS )6r@   z1Holds a packed distributed variable and a device.c                 C   s   || _ || _d S r<   )_var_device)r5   r   r   r
   r
   r   r3      s   
zPackedVarAndDevice.__init__c                 C   sJ   z t | j t| j|W  d    W S 1 sw   Y  W d S     r<   )r   r   r   getattrr   )r5   r   r
   r
   r   __getattr__   s   
(zPackedVarAndDevice.__getattr__c                 C   r;   r<   )r   r=   r
   r
   r   r     s   zPackedVarAndDevice.varc                 C   :   t | j | j W  d    S 1 sw   Y  d S r<   )r   r   r   r   rM   r=   r
   r
   r   rM        $zPackedVarAndDevice.valuec                 C   r   r<   )r   r   r   r   rb   r=   r
   r
   r   rb     r   zPackedVarAndDevice.read_valuec                 C   s   | j | jS r<   )r   r    r   r=   r
   r
   r   r      s   z PackedVarAndDevice.initial_valuec                 C   r   r<   )r   r   r   r   initialized_valuer=   r
   r
   r   r     r   z$PackedVarAndDevice.initialized_valuec                 C   r;   r<   )r   r=   r
   r
   r   r   !  r?   zPackedVarAndDevice.devicec                 C   8   t | j | jjW  d    S 1 sw   Y  d S r<   )r   r   r   r   r   r=   r
   r
   r   r   %     $zPackedVarAndDevice.handlec                 C   s<   t | j | j jW  d    S 1 sw   Y  d S r<   )r   r   r   r   rI   r   r=   r
   r
   r   on_device_handle*  s   
$z#PackedVarAndDevice.on_device_handlec                 C   r   r<   )r   r   r   r   opr=   r
   r
   r   r   .  r   zPackedVarAndDevice.opNTc                 C   B   t | j | j||||W  d    S 1 sw   Y  d S r<   )r   r   r   r   rZ   r5   rd   ra   r   rb   r
   r
   r   rZ   3     $zPackedVarAndDevice.assign_subc                 C   r   r<   )r   r   r   r   rg   r   r
   r
   r   rg   7  r   zPackedVarAndDevice.assign_addc                 C   r   r<   )r   r   r   r   ri   )r5   rM   ra   r   rb   r
   r
   r   ri   ;  r   zPackedVarAndDevice.assignFc                 C   @   t | j | j|||W  d    S 1 sw   Y  d S r<   )r   r   r   r   rl   r5   rn   ra   r   r
   r
   r   rl   ?     $zPackedVarAndDevice.scatter_subc                 C   r   r<   )r   r   r   r   rq   r   r
   r
   r   rq   C  r   zPackedVarAndDevice.scatter_addc                 C   r   r<   )r   r   r   r   rs   r   r
   r
   r   rs   G  r   zPackedVarAndDevice.scatter_mulc                 C   r   r<   )r   r   r   r   ru   r   r
   r
   r   ru   K  r   zPackedVarAndDevice.scatter_divc                 C   r   r<   )r   r   r   r   rw   r   r
   r
   r   rw   O  r   zPackedVarAndDevice.scatter_minc                 C   r   r<   )r   r   r   r   ry   r   r
   r
   r   ry   S  r   zPackedVarAndDevice.scatter_maxc                 C   r   r<   )r   r   r   r   r{   r   r
   r
   r   r{   W  r   z!PackedVarAndDevice.scatter_updatec                 C   sB   t | j | jj|||dW  d    S 1 sw   Y  d S r}   )r   r   r   r   r   r   r
   r
   r   r   [  s   $z'PackedVarAndDevice._dense_var_to_tensorc                 C   s
   | j  S r<   )r   _as_graph_elementr=   r
   r
   r   r   b  rC   z$PackedVarAndDevice._as_graph_elementr   r   r   )r   r   r   r   r3   r   r   rM   rb   r   r    r   r   r   r   r   rZ   rg   ri   rl   rq   rs   ru   rw   ry   r{   r   r   r
   r
   r
   r   r@      s:    














r@   NFc                 C   s   | j |||dS r}   )r   )r   r   r   r~   r
   r
   r   (_tensor_conversion_packed_var_and_devicef  s   r   r   )r   tensorflow.python.distributer   tensorflow.python.eagerr   tensorflow.python.frameworkr   r   tensorflow.python.opsr   r   BaseResourceVariabler   objectr@   r   #register_tensor_conversion_functionr
   r
   r
   r   <module>   s"    \s
