o
    i ep                     @   s   d Z ddlm  mZ ddlmZ ddlmZ ddl	m
Z
 ddl	mZ ddlmZ ddlmZ ed	e d
ZG dd dZG dd dZedg dG dd dejZdS )z>FeatureSpace structured data preprocessing & encoding utility.    N)backend)
base_layer)
saving_lib)serialization_lib)
LazyLoader)keras_exportlayerszkeras.src.layersc                   @   s6   e Zd ZdddZedd Zdd Zedd	 Zd
S )Crossone_hotc                 C   s0   |dvrt d| t|| _|| _|| _d S )N>   intr
   zdInvalid value for argument `output_mode`. Expected one of {'int', 'one_hot'}. Received: output_mode=)
ValueErrortuplefeature_namescrossing_dimoutput_mode)selfr   r   r    r   L/var/www/myenv/lib/python3.10/site-packages/keras/src/utils/feature_space.py__init__    s   

zCross.__init__c                 C   s   d | jS )N_X_)joinr   r   r   r   r   name+      z
Cross.namec                 C   s   | j | j| jdS )Nr   r   r   r   r   r   r   r   
get_config/   s   zCross.get_configc                 C      | di |S Nr   r   clsconfigr   r   r   from_config6      zCross.from_configNr
   )	__name__
__module____qualname__r   propertyr   r   classmethodr!   r   r   r   r   r	      s    

r	   c                   @   s(   e Zd Zdd Zdd Zedd ZdS )Featurec                 C   s@   |dvrt d| || _t|trt|}|| _|| _d S )N>   r   floatr
   zmInvalid value for argument `output_mode`. Expected one of {'int', 'one_hot', 'float'}. Received: output_mode=)r   dtype
isinstancedictr   deserialize_keras_objectpreprocessorr   )r   r+   r/   r   r   r   r   r   <   s   

zFeature.__init__c                 C   s   | j t| j| jdS )Nr+   r/   r   )r+   r   serialize_keras_objectr/   r   r   r   r   r   r   K   s   zFeature.get_configc                 C   r   r   r   r   r   r   r   r!   T   r"   zFeature.from_configN)r$   r%   r&   r   r   r(   r!   r   r   r   r   r)   ;   s
    	r)   zkeras.utils.FeatureSpace)v1c                   @   sd  e Zd ZdZedGddZedd ZedHdd	ZedIddZedHddZ	e	dJddZ
e				dKddZe				dKddZedLddZedLddZ					dMddZdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Zd1d2 Zd3d4 Zd5d6 Zd7d8 Zd9d: Zed;d< Zd=d> Zd?d@ Z dAdB Z!dCdD Z"dEdF Z#dS )NFeatureSpacea  One-stop utility for preprocessing and encoding structured data.

    Arguments:
        feature_names: Dict mapping the names of your features to their
            type specification, e.g. `{"my_feature": "integer_categorical"}`
            or `{"my_feature": FeatureSpace.integer_categorical()}`.
            For a complete list of all supported types, see
            "Available feature types" paragraph below.
        output_mode: One of `"concat"` or `"dict"`. In concat mode, all
            features get concatenated together into a single vector.
            In dict mode, the FeatureSpace returns a dict of individually
            encoded features (with the same keys as the input dict keys).
        crosses: List of features to be crossed together, e.g.
            `crosses=[("feature_1", "feature_2")]`. The features will be
            "crossed" by hashing their combined value into
            a fixed-length vector.
        crossing_dim: Default vector size for hashing crossed features.
            Defaults to `32`.
        hashing_dim: Default vector size for hashing features of type
            `"integer_hashed"` and `"string_hashed"`. Defaults to `32`.
        num_discretization_bins: Default number of bins to be used for
            discretizing features of type `"float_discretized"`.
            Defaults to `32`.

    **Available feature types:**

    Note that all features can be referred to by their string name,
    e.g. `"integer_categorical"`. When using the string name, the default
    argument values are used.

    ```python
    # Plain float values.
    FeatureSpace.float(name=None)

    # Float values to be preprocessed via featurewise standardization
    # (i.e. via a `keras.layers.Normalization` layer).
    FeatureSpace.float_normalized(name=None)

    # Float values to be preprocessed via linear rescaling
    # (i.e. via a `keras.layers.Rescaling` layer).
    FeatureSpace.float_rescaled(scale=1., offset=0., name=None)

    # Float values to be discretized. By default, the discrete
    # representation will then be one-hot encoded.
    FeatureSpace.float_discretized(
        num_bins, bin_boundaries=None, output_mode="one_hot", name=None)

    # Integer values to be indexed. By default, the discrete
    # representation will then be one-hot encoded.
    FeatureSpace.integer_categorical(
        max_tokens=None, num_oov_indices=1, output_mode="one_hot", name=None)

    # String values to be indexed. By default, the discrete
    # representation will then be one-hot encoded.
    FeatureSpace.string_categorical(
        max_tokens=None, num_oov_indices=1, output_mode="one_hot", name=None)

    # Integer values to be hashed into a fixed number of bins.
    # By default, the discrete representation will then be one-hot encoded.
    FeatureSpace.integer_hashed(num_bins, output_mode="one_hot", name=None)

    # String values to be hashed into a fixed number of bins.
    # By default, the discrete representation will then be one-hot encoded.
    FeatureSpace.string_hashed(num_bins, output_mode="one_hot", name=None)
    ```

    Examples:

    **Basic usage with a dict of input data:**

    ```python
    raw_data = {
        "float_values": [0.0, 0.1, 0.2, 0.3],
        "string_values": ["zero", "one", "two", "three"],
        "int_values": [0, 1, 2, 3],
    }
    dataset = tf.data.Dataset.from_tensor_slices(raw_data)

    feature_space = FeatureSpace(
        features={
            "float_values": "float_normalized",
            "string_values": "string_categorical",
            "int_values": "integer_categorical",
        },
        crosses=[("string_values", "int_values")],
        output_mode="concat",
    )
    # Before you start using the FeatureSpace,
    # you must `adapt()` it on some data.
    feature_space.adapt(dataset)

    # You can call the FeatureSpace on a dict of data (batched or unbatched).
    output_vector = feature_space(raw_data)
    ```

    **Basic usage with `tf.data`:**

    ```python
    # Unlabeled data
    preprocessed_ds = unlabeled_dataset.map(feature_space)

    # Labeled data
    preprocessed_ds = labeled_dataset.map(lambda x, y: (feature_space(x), y))
    ```

    **Basic usage with the Keras Functional API:**

    ```python
    # Retrieve a dict Keras Input objects
    inputs = feature_space.get_inputs()
    # Retrieve the corresponding encoded Keras tensors
    encoded_features = feature_space.get_encoded_features()
    # Build a Functional model
    outputs = keras.layers.Dense(1, activation="sigmoid")(encoded_features)
    model = keras.Model(inputs, outputs)
    ```

    **Customizing each feature or feature cross:**

    ```python
    feature_space = FeatureSpace(
        features={
            "float_values": FeatureSpace.float_normalized(),
            "string_values": FeatureSpace.string_categorical(max_tokens=10),
            "int_values": FeatureSpace.integer_categorical(max_tokens=10),
        },
        crosses=[
            FeatureSpace.cross(("string_values", "int_values"), crossing_dim=32)
        ],
        output_mode="concat",
    )
    ```

    **Returning a dict of integer-encoded features:**

    ```python
    feature_space = FeatureSpace(
        features={
            "string_values": FeatureSpace.string_categorical(output_mode="int"),
            "int_values": FeatureSpace.integer_categorical(output_mode="int"),
        },
        crosses=[
            FeatureSpace.cross(
                feature_names=("string_values", "int_values"),
                crossing_dim=32,
                output_mode="int",
            )
        ],
        output_mode="dict",
    )
    ```

    **Specifying your own Keras preprocessing layer:**

    ```python
    # Let's say that one of the features is a short text paragraph that
    # we want to encode as a vector (one vector per paragraph) via TF-IDF.
    data = {
        "text": ["1st string", "2nd string", "3rd string"],
    }

    # There's a Keras layer for this: TextVectorization.
    custom_layer = layers.TextVectorization(output_mode="tf_idf")

    # We can use FeatureSpace.feature to create a custom feature
    # that will use our preprocessing layer.
    feature_space = FeatureSpace(
        features={
            "text": FeatureSpace.feature(
                preprocessor=custom_layer, dtype="string", output_mode="float"
            ),
        },
        output_mode="concat",
    )
    feature_space.adapt(tf.data.Dataset.from_tensor_slices(data))
    output_vector = feature_space(data)
    ```

    **Retrieving the underlying Keras preprocessing layers:**

    ```python
    # The preprocessing layer of each feature is available in `.preprocessors`.
    preprocessing_layer = feature_space.preprocessors["feature1"]

    # The crossing layer of each feature cross is available in `.crossers`.
    # It's an instance of keras.layers.HashedCrossing.
    crossing_layer = feature_space.crossers["feature1_X_feature2"]
    ```

    **Saving and reloading a FeatureSpace:**

    ```python
    feature_space.save("myfeaturespace.keras")
    reloaded_feature_space = keras.models.load_model("myfeaturespace.keras")
    ```
    r
   c                 C   s   t |||dS )N)r   )r	   )r   r   r   r   r   r   r   cross   r"   zFeatureSpace.crossc                 C   s   t |||S N)r)   )r   r+   r/   r   r   r   r   feature$  r   zFeatureSpace.featureNc                 C   s<   ddl m} |ptd}|jd| dd}td|ddS )Nr   )identityr*   float32_preprocessor)r+   r   r0   )keras.src.layers.corer7   r   unique_object_nameIdentityr)   )r   r   r7   r/   r   r   r   r*   (  s   
zFeatureSpace.float      ?        c                 C   s2   |pt d}tj||| dd}td|ddS )Nfloat_rescaledr9   )scaleoffsetr   r8   r*   r0   )r   r;   r   	Rescalingr)   )r   r@   rA   r   r/   r   r   r   r?   4  s   zFeatureSpace.float_rescaledc                 C   s0   |pt d}tjd| dd}td|ddS )Nfloat_normalizedr9   )axisr   r8   r*   r0   )r   r;   r   Normalizationr)   )r   r   r/   r   r   r   rC   >     
zFeatureSpace.float_normalizedc                 C   s2   |pt d}tj||| dd}td||dS )Nfloat_discretizedr9   )num_binsbin_boundariesr   r8   r0   )r   r;   r   Discretizationr)   )r   rI   rJ   r   r   r/   r   r   r   rH   H  s   zFeatureSpace.float_discretized   c                 C   2   |pt d}tj| d||d}td||dS )Ninteger_categoricalr9   r   
max_tokensnum_oov_indicesint64r0   )r   r;   r   IntegerLookupr)   r   rP   rQ   r   r   r/   r   r   r   rN   V     z FeatureSpace.integer_categoricalc                 C   rM   )Nstring_categoricalr9   rO   stringr0   )r   r;   r   StringLookupr)   rT   r   r   r   rV   h  rU   zFeatureSpace.string_categoricalc                 C   0   |pt d}tj| d|d}td||dS )Nstring_hashedr9   r   rI   rW   r0   r   r;   r   Hashingr)   r   rI   r   r   r/   r   r   r   rZ   z  rG   zFeatureSpace.string_hashedc                 C   rY   )Ninteger_hashedr9   r[   rR   r0   r\   r^   r   r   r   r_     rG   zFeatureSpace.integer_hashedconcat    c           
         sd  |st d| _| _| _ fdd| D  _g  _|rat| }|D ]8}t	|t
r4t|}t	|tr@ j| q(|sFt d|D ]}	|	|vrUt d| qH jt||d q(dd  jD  _|dvrut d	| | _ fd
d j D  _dd  j D  _d  _ fdd jD  _i  _d _d _d  _d  _d  _d S )Nz0The `features` argument cannot be None or empty.c                       i | ]\}}|  ||qS r   )_standardize_feature.0r   valuer   r   r   
<dictcomp>      z)FeatureSpace.__init__.<locals>.<dictcomp>zzWhen specifying `crosses`, the argument `crossing_dim` (dimensionality of the crossing space) should be specified as well.zwAll features referenced in the `crosses` argument should be present in the `features` dict. Received unknown features: )r   c                 S   s   i | ]}|j |qS r   r   re   r4   r   r   r   rg         >   r-   r`   zdInvalid value for argument `output_mode`. Expected one of {'dict', 'concat'}. Received: output_mode=c                    rb   r   )_feature_to_inputrd   r   r   r   rg     rh   c                 S   s   i | ]\}}||j qS r   )r/   rd   r   r   r   rg     s    c                    s   i | ]	}|j  |qS r   )r   _cross_to_crosserrj   r   r   r   rg     s    F)r   r   hashing_dimnum_discretization_binsitemsfeaturescrossessetkeysr,   r-   r   r.   r	   appendcrosses_by_namer   inputspreprocessorsencoded_featurescrossersone_hot_encodersbuilt_is_adaptedr`   _preprocessed_features_names_crossed_features_names)
r   rq   r   rr   r   rn   ro   feature_setr4   keyr   r   r   r     sj   	






zFeatureSpace.__init__c                 C   s   t jd|j|dS )N)rL   )shaper+   r   )r   Inputr+   r   r   r6   r   r   r   rl        zFeatureSpace._feature_to_inputc                 C   s   t |tr|S t |trt|S |dkr| j|dS |dkr%| j|dS |dkr/| j|dS |dkr;| j|| j	dS |dkrE| j
|dS |dkrO| j|dS |d	kr[| j| j|dS |d
krg| j| j|dS td| )Nr*   ri   rC   r?   rH   r[   rN   rV   r_   rZ   zInvalid feature type: )r,   r)   r-   r   r.   r*   rC   r?   rH   ro   rN   rV   r_   rn   rZ   r   r   r   r   r   rc     s.   


z!FeatureSpace._standardize_featurec                 C   s   t j|j|jdS )Nri   )r   HashedCrossingr   r   )r   r4   r   r   r   rm     r   zFeatureSpace._cross_to_crosserc                 C   sN   g }| j  D ]}| j| }t|tjr|jd urqt|dr$|| q|S )Nadapt)	rq   rt   rx   r,   r   rF   
input_meanhasattrru   )r   adaptable_preprocessorsr   r/   r   r   r   _list_adaptable_preprocessors  s   



z*FeatureSpace._list_adaptable_preprocessorsc                    s   t |tjjstd| dt| d|  D ]5 | fdd}| j  }|	dD ]}q-|j
jdkr;|d}|j
jd	v rH|d
d }|| qd| _|   d| _d S )NzE`adapt()` can only be called on a tf.data.Dataset. Received instead: 
 (of type )c                    s   |   S r5   r   xri   r   r   <lambda>  s    z$FeatureSpace.adapt.<locals>.<lambda>rL   r   ra   >   r   rL   c                 S   s   t | dS )NrD   )tfexpand_dimsr   r   r   r   r   !  s    T)r,   r   dataDatasetr   typer   maprx   taker   rankbatchr   r}   get_encoded_featuresr|   )r   datasetfeature_datasetr/   r   r   ri   r   r     s.   


zFeatureSpace.adaptc                 C   s   |    | jS r5   )_check_if_builtrw   r   r   r   r   
get_inputs(  s   zFeatureSpace.get_inputsc                 C   s@   |    | jd u r| | j}| |}| ||}|| _| jS r5   )_check_if_adaptedry   _preprocess_featuresrw   _cross_features_merge_features)r   preprocessed_featurescrossed_featuresmerged_featuresr   r   r   r   ,  s   

z!FeatureSpace.get_encoded_featuresc                    s    fdd   D S )Nc                    s    i | ]}|j |  | qS r   )rx   re   r   rq   r   r   r   rg   9  s    z5FeatureSpace._preprocess_features.<locals>.<dictcomp>)rt   )r   rq   r   r   r   r   8  s   z!FeatureSpace._preprocess_featuresc                    sB   i }| j D ]} fdd|jD }| j|j |}|||j< q|S )Nc                       g | ]} | qS r   r   r   rq   r   r   
<listcomp>A  rk   z0FeatureSpace._cross_features.<locals>.<listcomp>)rr   r   rz   r   )r   rq   all_outputsr4   rw   outputsr   r   r   r   >  s   
zFeatureSpace._cross_featuresc                    sx  j st _ t  _j j }fddj D  fddjD  }jdkr3i }ng }jrjt||D ] \}}j|d }	|	rN|	|}jdkrX|||< q=|	| q=jdkre|S 
|S fddj D fddjD  }
t|||
D ]\}}}|jj}|jdkrj|pj|}d }|jjdstd	| d
| dt|tjtjfr| }n)t|tjr|j}nt|tjr|j}nt|tjtjfr|j}ntd	| d|d urtj|dd}	|	j|< |	|}jdkr#|jj}|ds|dkrtd| d| d|	| q|||< qjdkr:tjdd_

|S |S )Nc                    r   r   r   r   )r   r   r   r   P  s    z0FeatureSpace._merge_features.<locals>.<listcomp>c                    r   r   r   r   )r   r   r   r   S  rk   r-   c                       g | ]} j | qS r   r   r   r   r   r   r   k      
c                    r   r   )rv   r   r   r   r   r   m  r   r
   r   z	Feature 'zh' has `output_mode='one_hot'`. Thus its preprocessor should return an int64 dtype. Instead it returns a z dtype.z' has `output_mode='one_hot'`. However it isn't a standard feature and the dimensionality of its output space is not known, thus it cannot be one-hot encoded. Try using `output_mode='int'`.	multi_hot)
num_tokensr   r`   rW   z-Cannot concatenate features because feature 'z%' has not been encoded (it has dtype z'). Consider using `output_mode='dict'`.rD   rE   )r~   sortedrt   r   r   r|   zipr{   getru   r`   r+   r   rx   rz   
startswithr   r,   r   rS   rX   vocabulary_sizeCategoryEncodingr   rK   rI   r   r]   Concatenate)r   r   r   	all_namesall_featuresoutput_dictfeatures_to_concatr   r6   encoder	all_specsspecr+   r/   cardinalityr   )r   r   r   r   r   F  s   















zFeatureSpace._merge_featuresc                 C   s$   | j s|  sd| _ d S tdd S )NTzUYou need to call `.adapt(dataset)` on the FeatureSpace before you can start using it.)r}   r   r   r   r   r   r   r     s   
zFeatureSpace._check_if_adaptedc                 C   s$   | j s|   |   d| _ d S d S NT)r|   r   r   r   r   r   r   r     s
   
zFeatureSpace._check_if_builtc                 C   s$  |    t|tstd| dt| dd | D }d}| D ]%\}}|jjdkr;t	|ddg||< d}q$|jjdkrIt
|d	||< q$| |}| |}| ||}|r| jd
krq|jd dksjJ tj|ddS | D ]\}}|jjdkr|jd dkrtj|dd||< qu|S )Nz>A FeatureSpace can only be called with a dict. Received: data=r   c                 S   s   i | ]
\}}|t |qS r   )r   convert_to_tensor)re   r   rf   r   r   r   rg     s    z)FeatureSpace.__call__.<locals>.<dictcomp>Fr   rL   TrD   r`   r      )r   r,   r-   r   r   rp   r   r   r   reshaper   r   r   r   r   squeeze)r   r   	rebatchedr   r   preprocessed_datacrossed_datamerged_datar   r   r   __call__  s<   



zFeatureSpace.__call__c                 C   s*   t | j| jt | j| j| j| jdS )N)rq   r   rr   r   rn   ro   )r   r1   rq   r   rr   r   rn   ro   r   r   r   r   r     s   

zFeatureSpace.get_configc                 C   r   r   r   r   r   r   r   r!     r"   zFeatureSpace.from_configc                 C   s   dd | j  D S )Nc                 S   s   i | ]
\}}||j  qS r   )r/   get_build_config)re   r   r6   r   r   r   rg     s    
z1FeatureSpace.get_build_config.<locals>.<dictcomp>)rq   rp   r   r   r   r   r     s   zFeatureSpace.get_build_configc                 C   s.   |  D ]}| j| j||  qd| _d S r   )rt   rq   r/   build_from_configr}   )r   r    r   r   r   r   r     s   
zFeatureSpace.build_from_configc                 C   s   t | | dS )a  Save the `FeatureSpace` instance to a `.keras` file.

        You can reload it via `keras.models.load_model()`:

        ```python
        feature_space.save("myfeaturespace.keras")
        reloaded_feature_space = keras.models.load_model("myfeaturespace.keras")
        ```
        N)r   
save_model)r   filepathr   r   r   save  s   
zFeatureSpace.savec                 C      d S r5   r   r   storer   r   r   save_own_variables      zFeatureSpace.save_own_variablesc                 C   r   r5   r   r   r   r   r   load_own_variables  r   zFeatureSpace.load_own_variablesr#   r5   )r=   r>   N)Nr
   N)NrL   r
   N)r
   N)r`   Nra   ra   ra   )$r$   r%   r&   __doc__r(   r4   r6   r*   r?   rC   rH   rN   rV   rZ   r_   r   rl   rc   rm   r   r   r   r   r   r   r   r   r   r   r   r!   r   r   r   r   r   r   r   r   r   r3   Y   sv     F
			
G$f


r3   )r   tensorflow.compat.v2compatv2r   	keras.srcr   keras.src.enginer   keras.src.savingr   r   keras.src.utils.generic_utilsr    tensorflow.python.util.tf_exportr   globalsr   r	   r)   Layerr3   r   r   r   r   <module>   s   
