Sp update

.gitignore

@@ -4,4 +4,6 @@ build/
 __pycache__/
 codauto/__pycache__/
 codauto/.ipynb_checkpoints/
-pruebas_codifier.py
+pruebas_codifier.py
+*.venv
+.idea/

codauto/__init__.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 from .utils import *
 from .codifier import *
-from .prompt_maker import *
-from .prompt_text import *
 from .structured import *
 from .codifier_fasttext import *

codauto/codifier.py

@@ -40,6 +40,7 @@ class Codifier(ABC):
         train_df (pd.DataFrame): Cleaned training dataset.
         test_df (pd.DataFrame): Cleaned testing dataset with hierarchical labels.
         language (str): The language of your textual data.
+        preprocess (bool): Specifies whether the text from the training, test, or prediction set should be preprocessed.
 
     Methods:
         get_correspondences(corres_df):
@@ -96,15 +97,17 @@ class Codifier(ABC):
 
     def __init__(self,
                  structure_instance,
-                 train_df,
-                 test_df,
                  root_path,
+                 train_df=None,
+                 test_df=None,
                  corres_df=None,
                  min_lenght_texts=3,
-                 language='es'
+                 language='es',
+                 preprocess = True
                  ):
         self.root_path = root_path
         self.language = language
+        self.preprocess = preprocess
         os.makedirs(self.root_path, exist_ok=True)
         self.logger = logging.getLogger(f'CNAECodifier.{id(self)}')
         self.logger.setLevel(logging.INFO)
@@ -115,16 +118,16 @@ def __init__(self,
         stream_handler.setFormatter(formatter)
         if not self.logger.hasHandlers():
             self.logger.addHandler(stream_handler)
-        self.logger.propagate = False
+        self.logger.propagate = True
         self.min_lenght_texts = min_lenght_texts
         self.structure = structure_instance
         self.model = None
         if corres_df is not None:
             self.correspondences = self.get_correspondences(corres_df)
-        else:
-            self.correspondences = None
-        self.train_df = self.get_train_dataset(train_df, 'train_df')
-        self.test_df = self.get_test_dataset(test_df, 'test_df')
+        if train_df is not None:
+            self.train_df = self.get_train_dataset(train_df, 'train_df')
+        if test_df is not None:
+            self.test_df = self.get_test_dataset(test_df, 'test_df')
 
     def get_correspondences(self, correspondences_df):
         """
@@ -232,10 +235,11 @@ def clean_data(self, data_df):
         data_df[col_n_0] = data_df[col_n_0].apply(
             lambda n: self.get_code(str(n).replace('.', ''))
         )
-        data_df[col_n_1] = data_df[col_n_1].apply(preprocess_text, args=(self.language,))
-        data_df[col_n_1] = data_df[col_n_1].apply(
-            lambda desc: np.nan if len(desc) <= self.min_lenght_texts else desc
-        )
+        if self.preprocess:
+            data_df[col_n_1] = data_df[col_n_1].apply(preprocess_text, args=(self.language,))
+            data_df[col_n_1] = data_df[col_n_1].apply(
+                lambda desc: np.nan if len(desc) <= self.min_lenght_texts else desc
+            )
         return data_df
 
     def load_data(self, data_df, data_name='data_df'):
@@ -960,7 +964,7 @@ def predict(
         raw_predictions = self.get_pred_for_batch(
             desc_l_not_direct_recoding,
             idxs_not_direct_recoding,
-            True
+            self.preprocess
         )
 
         for idx, original_code in zip(