o
    i e                     @   sV   d Z ddlm  mZ ddlmZ dd Zdd Z	dd	 Z
d
d Zdd Zdd ZdS )zOptimizer utilities.    N)
tf_loggingc                 C   s   t | } t| }|r0tjj r$dd |D }tj tjjj	|}ntj j
t|fd}ng }g }d}| D ]\}}|du rH|d|f q8||| |f |d7 }q8|t|ks`J d|S )zReturns all-reduced gradients aggregated via summation.

    Args:
      grads_and_vars: List of (gradient, variable) pairs.

    Returns:
      List of (gradient, variable) pairs where gradients have been all-reduced.
    c                 S   s   g | ]}|d  qS )r    ).0pairr   r   I/var/www/myenv/lib/python3.10/site-packages/keras/src/optimizers/utils.py
<listcomp>$   s    z,all_reduce_sum_gradients.<locals>.<listcomp>)argsr   N   zFailed to add all gradients)listfilter_empty_gradientstf__internal__
distributestrategy_supports_no_merge_callget_replica_context
all_reduceReduceOpSUM
merge_call_all_reduce_sum_fnappendlen)grads_and_varsfiltered_grads_and_varsgradsreducedreduced_with_nonesreduced_posgvr   r   r   all_reduce_sum_gradients   s*   	



r    c                 C   s   t | } | s| S g }g }| D ]\}}|du r|| q|||f qt |}|s=dd | D f}td| d|  d|rJtddd |D  |S )	zDFilter out `(grad, var)` pairs that have a gradient equal to `None`.Nc                 S   s   g | ]\}}|j qS r   name)r   _r   r   r   r   r   L   s    z*filter_empty_gradients.<locals>.<listcomp>z(No gradients provided for any variable: z. Provided `grads_and_vars` is .zGradients do not exist for variables %s when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?c                 S   s   g | ]}|j qS r   r!   )r   r   r   r   r   r   V   s    )tupler   
ValueErrorloggingwarning)r   filteredvars_with_empty_gradsgradvarvariabler   r   r   r   <   s.   r   c                         du rdd S  fdd}|S )@Creates a gradient transformation function for clipping by norm.Nc                 S      | S Nr   r   r   r   r   <lambda>^       z+make_gradient_clipnorm_fn.<locals>.<lambda>c                    P   t tj tjjjtjjjjjfrtdtj  d fdd| D }|S )NzQ`clipnorm` is not supported with `CenteralStorageStrategy`. The strategy used is r$   c                    s    g | ]\}}t | |fqS r   )r   clip_by_normr   r   r   clipnormr   r   r   n   s    zKmake_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fn.<locals>.<listcomp>	
isinstancer   r   get_strategyexperimentalCentralStorageStrategycompatv1r&   r   clipped_grads_and_varsr8   r   r   gradient_clipnorm_fn`   s   
z7make_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   r9   rC   r   r8   r   make_gradient_clipnorm_fn[   s   rE   c                    r.   )r/   Nc                 S   r0   r1   r   r2   r   r   r   r3   y   r4   z2make_global_gradient_clipnorm_fn.<locals>.<lambda>c                    sh   t tj tjjjtjjjjjfrtdtj  dt	|  \}}t
| \}}tt	||}|S )NzX`global_clipnorm` is not supported with `CenteralStorageStrategy`. The strategy used is r$   )r;   r   r   r<   r=   r>   r?   r@   r&   zipclip_by_global_normr
   )r   r   	variablesclipped_gradsr#   rB   r8   r   r   rC   {   s   z>make_global_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   rD   r   r8   r    make_global_gradient_clipnorm_fnv      rJ   c                    r.   )zACreates a gradient transformation function for clipping by value.Nc                 S   r0   r1   r   r2   r   r   r   r3      r4   z,make_gradient_clipvalue_fn.<locals>.<lambda>c                    r5   )NzR`clipvalue` is not supported with `CenteralStorageStrategy`. The strategy used is r$   c                    s$   g | ]\}}t |   |fqS r   )r   clip_by_valuer7   	clipvaluer   r   r      s    zMmake_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fn.<locals>.<listcomp>r:   rA   rM   r   r   gradient_clipvalue_fn   s   
z9make_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fnr   )rN   rO   r   rM   r   make_gradient_clipvalue_fn   rK   rP   c                 C   s   | j tjjj|S r1   )extendedbatch_reduce_tor   r   r   r   )distributionr   r   r   r   r      s   
r   )__doc__tensorflow.compat.v2r?   v2r   tensorflow.python.platformr   r'   r    r   rE   rJ   rP   r   r   r   r   r   <module>   s   %