o
    i e                     @   s|   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z
 dd Zd	d
 Zdd Zdd Zdd Zdd Zdd ZdS )zOptimizer utilities.    )central_storage_strategy)distribute_lib)reduce_util)clip_ops)
tf_loggingc                 C   s   t | } t| }|r+t r dd |D }t jtjj	|}nt
 jt|fd}ng }g }d}| D ]\}}|du rC|d|f q3||| |f |d7 }q3|t|ks[J d|S )zReturns all-reduced gradients aggregated via summation.

  Args:
    grads_and_vars: List of (gradient, variable) pairs.

  Returns:
    List of (gradient, variable) pairs where gradients have been all-reduced.
  c                 S   s   g | ]}|d  qS )r    ).0pairr   r   Y/var/www/myenv/lib/python3.10/site-packages/tensorflow/python/keras/optimizer_v2/utils.py
<listcomp>%   s    z,all_reduce_sum_gradients.<locals>.<listcomp>)argsr   N   zFailed to add all gradients)listfilter_empty_gradientsstrategy_supports_no_merge_callr   get_strategyextended_replica_ctx_all_reduceds_reduce_utilReduceOpSUMget_replica_context
merge_call_all_reduce_sum_fnappendlen)grads_and_varsfiltered_grads_and_varsgradsreducedreduced_with_nonesreduced_posgvr   r   r
   all_reduce_sum_gradients   s*   	

r$   c                 C   s   t | } | s| S g }g }| D ]\}}|du r|| q|||f qt |}|s6tddd | D f |rCtddd |D  |S )zDFilter out `(grad, var)` pairs that have a gradient equal to `None`.Nz+No gradients provided for any variable: %s.c                 S   s   g | ]\}}|j qS r   name)r   _r#   r   r   r
   r   L   s    z*filter_empty_gradients.<locals>.<listcomp>zAGradients do not exist for variables %s when minimizing the loss.c                 S   s   g | ]}|j qS r   r%   )r   r#   r   r   r
   r   P   s    )tupler   
ValueErrorloggingwarning)r   filteredvars_with_empty_gradsgradvarr   r   r
   r   ;   s(   r   c                         du rdd S  fdd}|S )@Creates a gradient transformation function for clipping by norm.Nc                 S      | S Nr   r   r   r   r
   <lambda>W       z+make_gradient_clipnorm_fn.<locals>.<lambda>c                    4   t t tjtjfrtd fdd| D }|S )Nz:`clipnorm` is not supported with `CenteralStorageStrategy`c                    s    g | ]\}}t | |fqS r   )r   clip_by_normr   r"   r#   clipnormr   r
   r   a   s    zKmake_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fn.<locals>.<listcomp>
isinstancer   r   r   CentralStorageStrategyCentralStorageStrategyV1r)   r   clipped_grads_and_varsr:   r   r
   gradient_clipnorm_fnY   s   
z7make_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   r;   rB   r   r:   r
   make_gradient_clipnorm_fnT      rD   c                    r0   )r1   Nc                 S   r2   r3   r   r4   r   r   r
   r5   l   r6   z2make_global_gradient_clipnorm_fn.<locals>.<lambda>c                    sL   t t tjtjfrtdt|  \}}t	| \}}t
t||}|S )NzA`global_clipnorm` is not supported with `CenteralStorageStrategy`)r=   r   r   r   r>   r?   r)   zipr   clip_by_global_normr   )r   r   	variablesclipped_gradsr'   rA   r:   r   r
   rB   n   s   z>make_global_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   rC   r   r:   r
    make_global_gradient_clipnorm_fni   rE   rJ   c                    r0   )zACreates a gradient transformation function for clipping by value.Nc                 S   r2   r3   r   r4   r   r   r
   r5      r6   z,make_gradient_clipvalue_fn.<locals>.<lambda>c                    r7   )Nz;`clipvalue` is not supported with `CenteralStorageStrategy`c                    s$   g | ]\}}t |   |fqS r   )r   clip_by_valuer9   	clipvaluer   r
   r      s    
zMmake_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fn.<locals>.<listcomp>r<   r@   rL   r   r
   gradient_clipvalue_fn   s   
z9make_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fnr   )rM   rN   r   rL   r
   make_gradient_clipvalue_fn~   rE   rO   c                 C   s   | j tjj|S r3   )r   batch_reduce_tor   r   r   )distributionr   r   r   r
   r      s   r   c                  C   s    t  sdS t  } | j  S )zDReturns if the current Strategy can operate in pure replica context.T)r   has_strategyr   r   _use_merge_call)strategyr   r   r
   r      s   r   N)__doc__tensorflow.python.distributer   r   r   r   tensorflow.python.opsr   tensorflow.python.platformr   r*   r$   r   rD   rJ   rO   r   r   r   r   r   r
   <module>   s   #