o
    i ez                     @   s  d Z ddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl$m'Z' ddl$m(Z( ddl$m)Z) ddl$m*Z+ ddl$m,Z, ddl$m-Z- ddl$m.Z. ddl/m0Z0 dd l1m2Z2 dd!l1m3Z3 dd"l4m5Z5 dd#l4m6Z6 G d$d% d%e7Z8dOd'd(Z9dPd)d*Z:d+d, Z;d-d. Z<d/d0 Z=d1d2 Z>d3d4 Z?d5d6 Z@d7d8 ZAd9d: ZBd;ejCd<eDfd=d>ZEd;ejCd<eDfd?d@ZFd;ejCd<eDfdAdBZGG dCdD dDejHZIG dEdF dFejHZJG dGdH dHejHZKG dIdJ dJeIZLdKdL ZMdMdN ZNdS )Qz5Library for testing DistributionStrategy descendants.    N)
config_pb2)	event_pb2session)dataset_ops)collective_all_reduce_strategy)distribute_lib)distribute_utils)mirrored_strategy)reduce_util)tpu_strategy)backprop)context)def_function)test)dtypes)errors)ops)	test_util)	tf_record)	array_ops)gradients_impl)init_ops)init_ops_v2)math_ops)summary_ops_v2)variable_scope)variable_v1)	variables)gfile)	optimizer)training_util)nest)
tf_inspectc                   @   s   e Zd ZdS )_TestExceptionN)__name__
__module____qualname__ r(   r(   ]/var/www/myenv/lib/python3.10/site-packages/tensorflow/python/distribute/strategy_test_lib.pyr$   :   s    r$   Fc                 C   s   |rt  s| S t | S N)r   executing_eagerlyr   function)fnrun_in_functionr(   r(   r)   _maybe_run_in_function@   s   r/   c                 C   s   t  r*   )r$   _r(   r(   r)   _raise_exception_fnI   s   r2   c                   C      t  t d S r*   )r   get_replica_context
merge_callr2   r(   r(   r(   r)   _merge_raises_fnO      r6   c                 C      | j t d S r*   )extendedcall_for_each_replicar2   distr(   r(   r)   _call_raises_fnV      r=   c                   C   r3   r*   )r   r4   r5   r=   r(   r(   r(   r)   _merge_call_raises_fn]   r7   r?   c                 C   r8   r*   )r9   r:   r6   r;   r(   r(   r)   _call_merge_raises_fnd   r>   r@   c                   C   r3   r*   )r   r4   r5   r@   r(   r(   r(   r)   _merge_call_merge_raises_fnl   r7   rA   c                 C   sp   |  t| t|}| |d tttj	
||d }g }|D ]}t }|| || q%|S )z(Reads summary events from log directory.   r   )
assertTruer   ExistsListDirectory	assertLenlistr   tf_record_iteratorospathjoinr   EventParseFromStringappend)	test_caselogdirfilesrecordsresultreventr(   r(   r)   _events_from_logdirp   s   

rV   c                 C   s$   t jt ||d}tj|| ddS )zFUtitlity for create variables that works like variable in keras layer.)dtypeT)initial_valuename	trainable)	functoolspartialr   GlorotUniformr   Variable)rY   shaperW   initializerr(   r(   r)    create_variable_like_keras_layer~   s   
ra   c                 C   s&   t | j}d|jd t|j  v S )Nvar_list)r#   getfullargspecminimizeargslendefaults)optimizer_objarg_specr(   r(   r)   is_optimizer_v2_instance   s   rj   strategyreturnc                 C      t | tjtjfS r*   )
isinstancemirrored_libMirroredStrategyMirroredStrategyV1rk   r(   r(   r)   is_mirrored_strategy   s   
rs   c                 C   rm   r*   )rn   mwms_libCollectiveAllReduceStrategyCollectiveAllReduceStrategyV1rr   r(   r(   r)   !is_multi_worker_mirrored_strategy   s   rw   c                 C   s   t | tjtjtjfS r*   )rn   r   TPUStrategyTPUStrategyV1TPUStrategyV2rr   r(   r(   r)   is_tpu_strategy   s
   r{   c                   @   s|   e Zd ZdZdd Z		dddZdd	 Zd
d Zdd Zdd Z		dddZ
			dddZdd ZdddZdd ZdS ) DistributionTestBasez:Some tests that should work with any DistributionStrategy.c                    s      a tddtjdfdd}t|tdd t	dgg fd	d
}t
dD ]}| \}}|dkrC|\}|\}q5t| d }	t| d }
| |
|	 W d    d S 1 shw   Y  d S )NkernelrB   rB   rY   r_   rW   c                    &   t t|  g t d }|| S N      ?r   reshaper   mat_mulidentityxyr}   r(   r)   loss      z<DistributionTestBase._test_minimize_loss_eager.<locals>.lossc                 S   s   |  d| S )N皙?
assign_subvgr(   r(   r)   update      z>DistributionTestBase._test_minimize_loss_eager.<locals>.updater   c               
          j jfd} g }g }| D ]W\}} j |}|| t|g:  j jtjj	||d}t j j
||fdd | j | W d   n1 sRw   Y  W d   n1 saw   Y  q||fS zPerform one optimization step.re   )destinationsF)re   groupNr9   r:   read_varrN   r   control_dependencies	reduce_tor   ReduceOpSUMr   g_vbefore_list
after_listr   r   fetcheddgrad_fnoner   r(   r)   step   s&   

z<DistributionTestBase._test_minimize_loss_eager.<locals>.step
   r   rB   )scopera   r   float32r   implicit_gradr    get_filtered_grad_fnr   r   rangeabsnumpy
assertLess)selfr   r   r   ibabeforeaftererror_beforeerror_afterr(   )r   r   r}   r   r   r)   _test_minimize_loss_eager   s&   



"z.DistributionTestBase._test_minimize_loss_eagerFr   c                    s  t  }||_d|j_t  t 	  | j
|d|}  h tddtjdfdd}t|fdd	td
gg fdd}| \}}	t   tdD ]}
|||	f\}}|
dkrm|\}|\}q[t|d }t|d }| || W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Ng333333?)configr}   r~   r   c                    r   r   r   r   r   r(   r)   r      r   z<DistributionTestBase._test_minimize_loss_graph.<locals>.lossc                    s   |   | S r*   r   r   learning_rater(   r)   r      r   z>DistributionTestBase._test_minimize_loss_graph.<locals>.updater   c               
      r   r   r   r   r   r(   r)   r      s&   

z<DistributionTestBase._test_minimize_loss_graph.<locals>.stepr   r   rB   )r   ConfigProtoallow_soft_placementgpu_optionsper_process_gpu_memory_fractionr   
graph_moder   Graph
as_defaultcached_sessionr   ra   r   r   r   r   r   r   r   global_variables_initializerrunr   r   r   )r   r   soft_placementr   r   sessr   r   
before_out	after_outr   r   r   r   r   r   r   r(   )r   r   r}   r   r   r   r)   _test_minimize_loss_graph   sD   




 P z.DistributionTestBase._test_minimize_loss_graphc           	   
      s^  t  } fdd}|  }| z t f t }t	 s&|j
  td t| |j|}||}t	 sR|   || |   t| |}| |d | |d jjd jd | |d jjd jd W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                     sB       t j} td| W  d   S 1 sw   Y  dS )z#Function executed for each replica.r   N)r   r   r4   replica_id_in_sync_groupsummary_opswrite
replica_idsummary_writerr(   r)   run_fn  s   


$zHDistributionTestBase._test_summary_for_replica_zero_only.<locals>.run_fnr      rB   r   g        )tempfilemkdtempr   r   r   always_record_summariesr!   get_or_create_global_stepr   r+   r`   r   set_stepcreate_file_writerr9   r:   unwrapinitcloserV   rF   assertEqualsummaryvaluetagsimple_value)	r   r   rP   r   r   global_stepoutput	unwrappedeventsr(   r   r)   #_test_summary_for_replica_zero_only  s.   





P z8DistributionTestBase._test_summary_for_replica_zero_onlyc                    sv      - dgt jj  fdd} j| dgt jj  W d    d S 1 s4w   Y  d S )NFc                     s>    t j} | t jj |   d| < d S )NT)	evaluater   r4   r   r   rf   r9   worker_devicesassertFalser   r   expected_devicesr   r(   r)   mark_devices_fn3  s   z>DistributionTestBase._test_replica_id.<locals>.mark_devices_fnT)r   rf   r9   r   r:   assertAllEqual)r   r   r   r(   r   r)   _test_replica_id/  s   
"z%DistributionTestBase._test_replica_idc              	   C   s  |  } | t |jt W d    n1 sw   Y  | t |jt W d    n1 s6w   Y  | t |jt W d    n1 sQw   Y  | t |jt W d    n1 slw   Y  W d    d S W d    d S 1 sw   Y  d S r*   )	r   assertRaisesr$   r9   r:   r2   r6   r?   rA   )r   r<   r(   r(   r)   _test_call_and_merge_exceptions>  s    
"z4DistributionTestBase._test_call_and_merge_exceptionsc                    s    dg fdd}|S )Nr   c                    sj    |  | j | j dur!| j   S d | j d  d7  <   S )zInput fn for testing.Nr   rB   )assertIsNotNoner   num_replicas_in_syncnum_input_pipelinesinput_pipeline_id)input_contextdataset_or_callable_fnexpected_input_pipeline_idexpected_num_input_pipelinesexpected_num_replicas_in_syncr   worker_id_counterr(   r)   	_input_fnS  s   
zGDistributionTestBase._input_fn_to_test_input_context.<locals>._input_fnr(   )r   r   r   r   r   r   r(   r   r)   _input_fn_to_test_input_contextI  s   z4DistributionTestBase._input_fn_to_test_input_contextc           
      C   s   |r| j n| j}||}t rft|}|D ]}| t|t	|}	|||	 q| 
t | |t	| W d    n1 sEw   Y  t|}|D ]}| t|t	|}	|||	 qPd S t|}| j||jj|d|d d S )NTtest_reinitializeignore_order)assertCountEqualr   !distribute_datasets_from_functionr   r+   iterr   rG   experimental_local_resultsnextr   StopIterationr   make_initializable_iterator_test_input_fn_iteratorr9   r   )
r   rk   input_fnexpected_valuesr  assert_sameiterableiteratorexpected_valuecomputed_valuer(   r(   r)   _test_input_fn_iterablee  s0   


z,DistributionTestBase._test_input_fn_iterableNTc           
         s  fdd}||j  |D ]$}|  | fddtt|D }	|r,||	 q||	 qtj |  | fddtt|D  W d    n1 sWw   Y  |r||j  |D ]&}|  | fddtt|D }	|r||	 qe||	 qed S d S )Nc                    s   r | S  | S r*   )r   r   r   )r   r   r(   r)   <lambda>      z>DistributionTestBase._test_input_fn_iterator.<locals>.<lambda>c                       g | ]}t | qS r(   r	   select_replica.0rT   next_elementr(   r)   
<listcomp>  r  z@DistributionTestBase._test_input_fn_iterator.<locals>.<listcomp>c                    r  r(   r  r  r  r(   r)   r    r  c                    r  r(   r  r  r  r(   r)   r    s    )	r`   get_nextr   rf   r  r   r   r   OutOfRangeError)
r   r  devicesr  r   r  r  r   r  r  r(   )r  r   r   r)   r    s@   






z,DistributionTestBase._test_input_fn_iteratorc              	      s   |  J tjdg tjt dtjj	d | 
t   fdd}|j|\}}| 
|| ||}| 
|}| dt| | W d    d S 1 sQw   Y  d S )Nr   F)r_   rW   r`   rZ   aggregationc                     s     d}   }| |fS NrB   )
assign_add
read_value)train_opr   r   r(   r)   model_fn  s   
z?DistributionTestBase._test_global_step_update.<locals>.model_fn)rB   )r   r   get_variabler   int64r   zeros_initializerr   VariableAggregationONLY_FIRST_REPLICAr   r   r9   r:   r   r  r   rf   )r   rk   r(  	train_opsr   global_step_tensorsglobal_step_valuesr(   r'  r)   _test_global_step_update  s"   


"z-DistributionTestBase._test_global_step_updatec                    s  t |tjsd |p }|  |}tddgddgddgddgd	dgdd
gg}tg d}d}|jj	sB||j
 }|jj||f|pM d}	|	d}	|jj}
|	j||
d}	||	}|j  fdd}|||\}}|| || |||\}}|| || tj ||| W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nzn/a: V1 onlyrB   r               r         )r5  r4  r6  r   rB   r   r   )drop_remainderc                    sL   |  tdd  |\}}| || |f\}}t|t|fS )Nc                 S   s   | S r*   r(   )zr(   r(   r)   r    s    zWDistributionTestBase._test_numpy_dataset.<locals>.run_and_concatenate.<locals>.<lambda>)experimental_runr/   r   r  npconcatenate)rk   r   r   r   r.   r   r(   r)   run_and_concatenate  s   
zEDistributionTestBase._test_numpy_dataset.<locals>.run_and_concatenate)rn   r   
StrategyV1skipTestr   r   r;  asarrayr9   _global_batch_sizer   experimental_make_numpy_datasetrepeat"experimental_require_static_shapesbatchmake_dataset_iteratorr   r`   r   r   r   r   )r   rk   r   r.   r   r   r   r   
batch_sizedsr8  r   r>  x_1y_1x_2y_2r(   r=  r)   _test_numpy_dataset  s:   
.


Pz(DistributionTestBase._test_numpy_datasetc              	   C   s   t jtjfD ]P}| B |d}| d|j |dtjjd}| d|j |dtjjdd}| d|j |dtjjdd}| d|j W d    n1 sQw   Y  qd S )Nr   T)synchronizationF)rO  rZ   )	r   
VariableV1r   r^   r   r   rZ   VariableSynchronizationON_READ)r   rk   clsv1v2v3v4r(   r(   r)   _test_trainable_variable  s"   


z-DistributionTestBase._test_trainable_variable)Fr   F)NTF)NF)r%   r&   r'   __doc__r   r   r   r   r   r   r  r  r1  rN  rX  r(   r(   r(   r)   r|      s&    3
:"
!
)
&r|   c                   @   sp   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )OneDeviceDistributionTestBasezESome tests that should work with any one-device DistributionStrategy.c                 C   s   | dd }| dg| || |j dd |fd}| t|j|}| dg|d  | dg|d	  |j d
d |d}| dg| || d S )Nc                   S   s
   t dS )N      @)r   r   r(   r(   r(   r)   r    s   
 z9OneDeviceDistributionTestBase._test_run.<locals>.<lambda>r\  c                 S      | d | |  dS Nr   )r   r   r(   r  r(   r(   r)   r         r   g       @r   g      0@r   c                 S      |d|   d S Nr   r(   r   r   r(   r(   r)   r        kwargs      E@)r   r   r   r   r"   map_structure)r   rk   out1out2	out2_valsout3r(   r(   r)   	_test_run  s   z'OneDeviceDistributionTestBase._test_runc                 C   &   | j |tdddgfdddgfd d S )Nr\  rf       E@inputsexpected_test_collective_comms_all_sumr   rk   r(   r(   r)   _test_all_reduce_sum     
z2OneDeviceDistributionTestBase._test_all_reduce_sumc                 C      | j |tdgdgd d S Nr\  rp  expected_grads _test_collective_comms_gradientsrt  ru  r(   r(   r)   _test_all_reduce_sum_gradients     
z<OneDeviceDistributionTestBase._test_all_reduce_sum_gradientsc                 C   rx  ry  $_test_collective_comms_gradient_tapert  ru  r(   r(   r)   "_test_all_reduce_sum_gradient_tape  r  z@OneDeviceDistributionTestBase._test_all_reduce_sum_gradient_tapec                 C   rm  )N       @      5@g      6@ro  rs  	_all_meanru  r(   r(   r)   _test_all_reduce_mean  rw  z3OneDeviceDistributionTestBase._test_all_reduce_meanc                 C   rx  Ng      @rz  r}  r  ru  r(   r(   r)   _test_all_reduce_mean_gradients  r  z=OneDeviceDistributionTestBase._test_all_reduce_mean_gradientsc                 C   rx  r  r  r  ru  r(   r(   r)   #_test_all_reduce_mean_gradient_tape  r  zAOneDeviceDistributionTestBase._test_all_reduce_mean_gradient_tapec              	      sn   |  fdd |    | tt|j|| }| |d g|d  | |d g|d  d S )Nc                       t j S r*   r   Datasetfrom_tensorsr0   rp  r(   r)   r  "      zFOneDeviceDistributionTestBase._test_collective_comms.<locals>.<lambda>r   rB   )make_input_fn_iteratorr   
initializerG   mapr  r:  r   )r   rk   comm_fnrp  rq  outputsr(   r  r)   rs     s   

z4OneDeviceDistributionTestBase._test_collective_commsc                    sb   t  r	| d  fdd}|fdd|   | || ||| d S )Nz5`tf.gradients` is not supported with eager execution.c                    (   t d} ||  }t||gd S Nrf  r   r   r   r   	gradientscr   r   r  r(   r)   r   1     
zLOneDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.stepc                    r  r*   r  r0   r  r(   r)   r  7  r  zPOneDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.<lambda>)	r   r+   r@  r  r   r  r   r  r:  r   rk   r  rp  r{  r   r(   r  rp  r)   r}  ,  s   


z>OneDeviceDistributionTestBase._test_collective_comms_gradientsc                    sP    fdd}| fdd|   | || ||| d S )Nc                    T   t d}t }||  ||  }W d    n1 sw   Y  |||S Nrf  r   r   r   GradientTapewatchgradientr  r   taper   r  r(   r)   r   C     


zPOneDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.stepc                    r  r*   r  r0   r  r(   r)   r  K  r  zTOneDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.<lambda>)r  r   r  r   r  r:  r  r(   r  r)   r  @  s   

zBOneDeviceDistributionTestBase._test_collective_comms_gradient_tapec                 C   s   t  r	| d tdd\}}|dd }dd }|||}tj|d j	d}|
|  |
| W d    d S 1 sCw   Y  d S )	N:cross-device tests are not supported with eager execution.r   r   c                 S      t jdS Nr5  r   r  r   r0   r(   r(   r)   r  Z  r  z[OneDeviceDistributionTestBase._test_device_and_input_device_are_colocated.<locals>.<lambda>c                 S      | d S r#  r(   r  r(   r(   r)   r  [      rB   target)r   r+   r@  r   create_local_clusterr  r:  session_libSessionr  r   r  )r   rk   workersr1   rp  r  run_opr   r(   r(   r)   +_test_device_and_input_device_are_colocatedT  s   "zIOneDeviceDistributionTestBase._test_device_and_input_device_are_colocatedc           	      C   s   t  r	| d tdd\}}|dd }dd }t |j}t	
d |||}W d    n1 s8w   Y  tj|d jd	}||  || W d    d S 1 s^w   Y  d S )
Nr  r   r   c                 S   r  r  r  r0   r(   r(   r)   r  g  r  ziOneDeviceDistributionTestBase._test_device_and_input_device_are_colocated_with_function.<locals>.<lambda>c                 S   r  r#  r(   r  r(   r(   r)   r  h  r  z)/job:worker/replica:0/task:1/device:CPU:0rB   r  )r   r+   r@  r   r  r  r   r,   r:  r   devicer  r  r  r   r  )	r   rk   r  r1   rp  r  r:  r  r   r(   r(   r)   9_test_device_and_input_device_are_colocated_with_functiona  s"   "zWOneDeviceDistributionTestBase._test_device_and_input_device_are_colocated_with_functionN)r%   r&   r'   rZ  rl  rv  r~  r  r  r  r  rs  r}  r  r  r  r(   r(   r(   r)   r[    s    r[  c                   @   s|   e Zd ZdZdddZdddZdddZdd	d
ZdddZdddZ		dddZ
	dddZ	dddZ	dddZdS )TwoDeviceDistributionTestBasezESome tests that should work with any two-device DistributionStrategy.Fc                 C   s   | tdd |}| ddg| || |j tdd ||fd}| t|j|}| ddg|d  | ddg|d	  |j td
d ||d}| ddg| || d S )Nc                   S   s   t  jd S r#  )r   r4   r   r(   r(   r(   r)   r  w  s    z9TwoDeviceDistributionTestBase._test_run.<locals>.<lambda>rB   r   c                 S   r]  r^  r(   r  r(   r(   r)   r  |  r_  r   r4  r   r   c                 S   r`  ra  r(   rb  r(   r(   r)   r    rc  rd  r2     )r   r/   r   r   r   r"   rg  )r   rk   r.   rh  ri  rj  rk  r(   r(   r)   rl  u  s&    z'TwoDeviceDistributionTestBase._test_runc                 C   s4   | j |tddgddgddggfdddgf|d	 d S )
Nr         @     C@r       D@r\  rf  rn  rp  rq  r.   rr  r   rk   r.   r(   r(   r)   rv       

z2TwoDeviceDistributionTestBase._test_all_reduce_sumc                 C       | j |tddgddg|d d S Nr   r  r\  rp  r{  r.   r|  r  r(   r(   r)   r~       
z<TwoDeviceDistributionTestBase._test_all_reduce_sum_gradientsc                 C   r  r  r  r  r(   r(   r)   r    r  z@TwoDeviceDistributionTestBase._test_all_reduce_sum_gradient_tapec                 C   s4   | j |tddgddgddggfdddgf|d d S )	Nr   r  r  r  r  r  g     5@r  r  r  r(   r(   r)   r    r  z3TwoDeviceDistributionTestBase._test_all_reduce_meanc                 C   r  Nr   r  r  r  r  r  r(   r(   r)   r    r  z=TwoDeviceDistributionTestBase._test_all_reduce_mean_gradientsc                 C   r  r  r  r  r(   r(   r)   r    s   
zATwoDeviceDistributionTestBase._test_all_reduce_mean_gradient_tapec              
      s   |  fdd |    | tt|j|t|| }| |d |d g|d  | |d |d g|d  d S )Nc                    r  r*   r   r  from_tensor_slicesr0   r  r(   r)   r    r  zFTwoDeviceDistributionTestBase._test_collective_comms.<locals>.<lambda>r   rB   )	r  r   r  rG   r  r  r:  r/   r   )r   rk   r  rp  rq  r.   r  r(   r  r)   rs    s   

 z4TwoDeviceDistributionTestBase._test_collective_commsc                    sl   t  r|s| d  fdd}|fdd|   | || ||t	|| d S )NzP`tf.gradients` is not supported with eager execution without using tf.functions.c                    r  r  r  r  r  r(   r)   r     r  zLTwoDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.stepc                    r  r*   r  r0   r  r(   r)   r    r  zPTwoDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.<lambda>)
r   r+   r@  r  r   r  r   r  r:  r/   r   rk   r  rp  r{  r.   r   r(   r  r)   r}    s"   


z>TwoDeviceDistributionTestBase._test_collective_comms_gradientsc                    sV    fdd}| fdd|   | || ||t|| d S )Nc                    r  r  r  r  r  r(   r)   r     r  zPTwoDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.stepc                    r  r*   r  r0   r  r(   r)   r    r  zTTwoDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.<lambda>)r  r   r  r   r  r:  r/   r  r(   r  r)   r    s    
zBTwoDeviceDistributionTestBase._test_collective_comms_gradient_tapeNrY  )r%   r&   r'   rZ  rl  rv  r~  r  r  r  r  rs  r}  r  r(   r(   r(   r)   r  r  s     








r  c                   @   s@   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dS )&RemoteSingleWorkerMirroredStrategyBasez!Tests for a Remote single worker.c                 C   s   d S r*   r(   )r   r(   r(   r)   _get_num_gpus  s   z4RemoteSingleWorkerMirroredStrategyBase._get_num_gpusc                 C   s   |  |  |j d S r*   )r   r  r   r   distributionr(   r(   r)   _testNumReplicasInSync  s   z=RemoteSingleWorkerMirroredStrategyBase._testNumReplicasInSyncc                 C   s(   t  r| | d S | j|dd d S )Ng?r   )r   r+   r   r   r  r(   r(   r)   _testMinimizeLoss  s   z8RemoteSingleWorkerMirroredStrategyBase._testMinimizeLossc              	   C   s   |  ? td}td td}W d    n1 sw   Y  t r+d}nd}| |j| | |jd W d    d S 1 sFw   Y  d S )Nr   z/cpu:0z)/job:worker/replica:0/task:0/device:CPU:0z/job:worker/replica:0/task:0)r   r   r   r   r  r   r+   r   )r   r  r   r   r  r(   r(   r)   _testDeviceScope  s   

"z7RemoteSingleWorkerMirroredStrategyBase._testDeviceScopec                    s   dd }|    d fddtdd D }|  #}| j|  d d}||}| ||jj|| W d    d S 1 sBw   Y  d S )	Nc                   S   r  Nd   r  r(   r(   r(   r)   r    r  z\RemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset.<locals>.<lambda>rB   c                    s&   g | ]  fd dt D  qS )c                       g | ]} | qS r(   r(   r  jr   r(   r)   r        ziRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset.<locals>.<listcomp>.<listcomp>)r   )r  num_gpusnum_workersr  r)   r    s    z^RemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset.<locals>.<listcomp>r   r  r   r   r   )r  r   r   r   r  r  r9   r   )r   r  
dataset_fnr  r   r  r  r(   r  r)   #_testMakeInputFnIteratorWithDataset  s$   


"zJRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDatasetc           	   	      s   dd }|   }d}g }tdd|D ] | fddt|D |  q|  &}| j||| |d d}||}| j||jj||d	d
d W d    d S 1 sRw   Y  d S )Nc                  S   s   t jd} t | }|jS r  )r   r  r   make_one_shot_iteratorr  )datasetitr(   r(   r)   r-     s   
zWRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallable.<locals>.fnrB   r   r  c                    r  r(   r(   r  r  r(   r)   r  !  r  z_RemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallable.<locals>.<listcomp>r  FTr  )	r  r   rN   r   r   r  r  r9   r   )	r   r  r-   r  r  r  r   r  r  r(   r  r)   $_testMakeInputFnIteratorWithCallable  s&   "

"zKRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallableN)
r%   r&   r'   rZ  r  r  r  r  r  r  r(   r(   r(   r)   r    s    r  c                 C      t  }|tjj| S r*   )r   r4   
all_reducer   r   r   r   ctxr(   r(   r)   rt  2     rt  c                 C   r  r*   )r   r4   r  r   r   MEANr  r(   r(   r)   r  7  r  r  rY  r*   )OrZ  r[   rI   r   r   r;  tensorflow.core.protobufr   tensorflow.core.utilr   tensorflow.python.clientr   r  tensorflow.python.data.opsr   tensorflow.python.distributer   rt   r   r	   r
   ro   r   r   tensorflow.python.eagerr   r   r   r   tensorflow.python.frameworkr   r   r   r   tensorflow.python.lib.ior   tensorflow.python.opsr   r   r   r   r   r   r   r   r   r   tensorflow.python.platformr   tensorflow.python.trainingr    r!   tensorflow.python.utilr"   r#   	Exceptionr$   r/   r2   r6   r=   r?   r@   rA   rV   ra   rj   Strategyboolrs   rw   r{   TestCaser|   r[  r  r  rt  r  r(   r(   r(   r)   <module>   s   

	
  \ytL