
    
gi                         d dl Z d dlZd dlZd dlZd dlZd dlZda	da
dadaddededefdZd Zd Zd	 Zd
 Zd Zd Zd Zdeej        ej        f         fdZd Zd Zd ZdS )    N 
error_codeerror_messageadditional_datac                    	 t          d|  d|            | |d}|r|                    |           t          dd          5 }t          j        ||           d d d            n# 1 swxY w Y   t          j        |            d S # t          $ r1}t          d|            t          j        |            Y d }~d S d }~ww xY w)NzError z: )	errorCodeerrorMessageoutput.jsonwzFailed to log error: )printupdateopenjsondumpsysexit	Exception)r   r   r   
error_dataoutfilees         Z/var/www/dev.workforce-alpha.com/api/wfa-api/forecast-prediction-helpers/predict_helper.pyexitWithErrorr      s>   4z44]44555 $)
 

  	/o... -%% 	+Ij'***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	   )a))***s<   AB A%B %A))B ,A)-B 
C&B<<Cc            
         t          d           t          t          j                  dk    r#t          d           t          j        d           t          j        d         } 	 t          j        |            n# t          $ r* t          d|  d           t          j        d           Y nt          $ r* t          d|  d           t          j        d	           Y nMt          $ rA}t          d
t          |           d|  d           t          j        d           Y d }~nd }~ww xY wt                      }d|vrt          dd           |d         }|                    d           d}t          |          }t          ||          \  }}d|v r|d         }|                    dd            nd}d|v r	|d         }	nd }	|                    dd            d|v r	|d         }
nd}
|                    dd            d|vrd}n|d         }|                    dd            |
r t!          j        |d         dz             |d<   |||||||	|
||f
S )Nz"--Initializing prediction module--   z0Error: Missing argument: Path to temp directory.   zError: Directory  does not exist.z-Error: Permission denied to access directory .   An unexpected error occurred: z while accessing the directory    resultId   z'Error: 'resultId' parameter is missing.r
   onlyModelAnalysisFcutoff_minimumlog_transformanomaly_threshold_n_stddevg      @y)r   lenr   argvr   oschdirFileNotFoundErrorPermissionErrorr   strpredict_loadConfigr   poppredict_getRegressorspredict_loadDatanplog)input_directoryr   configr!   output_data_file
regressorshistData
futureDatar#   r$   r%   r&   s               r   predict_initializer;   ,   s   	
.///
38}}@AAAhqkO

!!!!   C/CCCDDD   PoPPPQQQ   hs1vvhhVehhhiii  !!FaBCCCj!H
JJz$&v..J+FJ??Hjf$$"#67

&----!6!! 01
JJ&&& &  /
JJ%%% (611%("" &,,H%I"
JJ+T222  2x}q011 8-z8ZQ_an  qK  M^  ^  ^s$   #A8 81D'+0D'	D'&7D""D'c            	      f   t          d           d} 	 t          |           5 }|                                }d d d            n# 1 swxY w Y   ny# t          $ r t	          dd|  d           Y nYt
          $ r t	          dd|  d           Y n:t          $ r.}t	          d	d
t          |           d|  d           Y d }~nd }~ww xY w	 t          j	        |          }nS# t          $ r t	          dd           Y n7t          $ r+}t	          dd
t          |           d           Y d }~nd }~ww xY w|S )Nz--Loading config--zconfig.json
   zFile r      z&Permission denied to open config file r      r   z* when opening and loading the config file    z5Could not decode JSON from the specified config file.   z* while decoding the specified config file.)r   r   readr,   r   r-   r   r.   r   loads
ValueError)input_config_filefsinput_config_json_stringr   r6   s        r   r/   r/   s   s#   	
%C#$$ 	1')wwyy$	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 G G GbE"3EEEFFFFF Y Y YbWCTWWWXXXXX C C Cb  B3q66  B  Bm~  B  B  B  	C  	C  	C  	C  	C  	C  	C  	CCo455 T T TbRSSSSS o o obm3q66mmmnnnnnnnno Msc   A AA AA 
AA C/C	C$C  C	C D.:	D.!D))D.c                 b   t          d           d| vrt          dd           	 | d         }|                     d           t          d           |D ]}t          d|            |S # t          $ r t          dd           Y d S t          $ r}t          dd	|            Y d }~d S d }~ww xY w)
Nz--getRegressors--r8      z,Error: 'regressors' key not found in config.zRegressors loaded:z        Runtime error: )r   r   r0   KeyErrorr   )r6   r8   	regressorr   s       r   r1   r1      s   	
6!!bHIII1L)


<   "#### 	$ 	$I"y""#### J J JbHIIIIII 1 1 1b/A//0000000001s   AA* *B.	B.B))B.c                 H   g }|D ]7}|d         }||dk     r|d         }t          d| d| d           t          j        t          |           t                    }| j        | |         dk                                             }t          t          |                    D ]k}||         dz   }	|dz   t          |          k     r||dz            nt          |           }
t          ||
|	z
            }t          d|dz             ||	|	|z   <   l|| d| <   d|d          d	d	d
}|	                    |           9|
                    |           t          d           t          |           t          d           t          |                     d                     d S )NnumDaysSincer   namez.Generating 'days since' column for regressor: , with a maximum of  days.dtypeds_r   rQ   rP   numDaysBeforez&New Regressors added for 'days since':zEFirst 10 rows of the dataframe after generating 'days since' columns:r=   )r   r3   zerosr(   intindextolistrangeminappendextendhead)dfr8   new_regressorsrN   rP   regressorNamerecords_since_1ones_indicesi	start_idxend_idxlengthnew_regressors                r   #predict_generateRegressorsDaysSincerl      s   N - -	 0<!#3#3!&)v}vvbnvvvwww (3r77#666 x= 1Q 67>>@@ s<(()) 	Q 	QA$Q!+I-.US5F5F-F-Fl1q5))CPRGGGw':;;F<A!VaZ<P<POIi&&8899 %4   ! ";y'8!:!:A`abbm,,,,n%%%	
2333	.	
QRRR	"''"++    c                    g }|D ]}|d         }||dk     r|d         }t          d| d| d           t          j        t          |           t                    }| j        | |         dk                                             }t          t          |                    D ]8}||         }	t          d|	|z
            }
|	|
z
  }t          |dd	          ||
|	<   9|| d
| <   d
|d          ddd}|	                    |           |
                    |           t          d           t          |           t          d           t          |                     d                     d S )NrX   r   rQ   z/Generating 'days before' column for regressor: rR   rS   rT   r   db_rW   z'New Regressors added for 'days before':zFFirst 10 rows of the dataframe after generating 'days before' columns:r=   )r   r3   rY   r(   rZ   r[   r\   r]   maxr_   r`   ra   )rb   r8   rc   rN   rX   rd   records_before_1rf   rg   ri   rh   rj   rk   s                r   $predict_generateRegressorsDaysBeforers      s   N - -	!/2 MA$5$5!&)xxxcpxxxyyy 8CGG3777 x= 1Q 67>>@@ s<(()) 	G 	GA"1oGAw677Iy(F2722F2FYw.// %5   ! ";y'8!:!:A`abbm,,,,n%%%	
3444	.	
RSSS	"''"++rm   c                 ~  	 	 g }g 	| j         dd          }|D ]o}| |                                                                         dk    r=|                    |           |D ]%}|d         |k    r	                    |            n&p|r.|                     |d           |                    |d           	fd|D             |d d <   t          d|           t          d	           d S # t          $ r}t          d	d
| d           Y d }~d S d }~wt          $ r}t          dd|            Y d }~d S d }~ww xY w)Nr   r   rQ   T)columnsinplacec                     g | ]}|v|	S  rx   ).0rregressors_to_removes     r   
<listcomp>z7predict_dropRegressorsWithNaNValues.<locals>.<listcomp>  s$    PPPq!;O2O2O2O2O2Orm   z&Columns dropped from both data frames:zRegressors dropped:   z
KeyError: z( - The specified column might not exist.   rL   )	ru   isnasumr_   dropr   rM   r   r   )
rb   df2r8   columns_to_dropcolumns_to_checkcolumnrN   ker   r{   s
            @r   #predict_dropRegressorsWithNaNValuesr      s   1 !:abb> ' 	 	F&z  $$&&**&&v...!+  I (F22,33I>>> 3  	<GGOTG:::HH_dH;;; QPPPJPPP
111 	6HHH#%9::::: U U UbSrSSSTTTTTTTTT 1 1 1b/A//0000000001s$   C(C- -
D<7DD<D77D<c                 X   t          d| j                   t          d           t          |                     d                     d| j        v r7|                     ddgd           t          d| j        d	                     n5|                     dgd           t          d
| j        d	                     t          d           t          j        | d         dd          | d<   | d                                                                         }|d	k    r+t          d| d           |                     dgd           d| j        v r|t          j	        | d         d          | d<   | d                                                                         }|d	k    r+t          d| d           |                     dgd           t          d| j                   t          d|                                                                                                             d S )NzInitial dataframe shape:zFirst 10 rows of the dataframe:r=   r'   dsT)subsetrv   z3Rows after dropping missing values in 'ds' or 'y': r   z,Rows after dropping missing values in 'ds': z*Converting 'ds' column to datetime format.%Y-%m-%d %H:%M:%Scoerce)formaterrorsz	Warning: z[ values could not be converted to datetime and were set to NaT. These rows will be removed.)r   z> non-numeric values in 'y' column. These rows will be removed.z.Final dataframe shape after all preprocessing:z/Total missing values in the cleaned dataframe: )
r   shapera   ru   dropnapdto_datetimer   r   
to_numeric)rb   	nat_countnumeric_y_nan_counts      r   predict_prepareDataframer     s.   	
$bh///	
+,,,	"''"++ bj
		$d	333QBHQKQQRRRR
		$	...JRXa[JJKKK 

6777~bh/B8TTTBtH 4##%%I1}}  A)  A  A  A  	B  	B  	B
		$	... bj-39993 gllnn0022""q1qqqrrrIIcUDI111 

:BHEEE	
SBGGIIMMOO<O<O<Q<Q
S
STTTTTrm   returnc                 Z   d| vrd}n| d         }d| vrt          dd           | d         a|                     d           d| vrt          dd           | d         a|                     d           d	| vrt          d
d           | d	         a|                     d	           d| vrt          dd           | d         a|                     d           d| vrt          dd           | d         }|                     d           |s1d| vrt          dd           | d         }|                     d           	 t          j                            t          t          t          t
                    }nt# t          j        j	        $ r*}t          ddt          |                      Y d }~n;d }~wt          $ r+}t          ddt          |           d           Y d }~nd }~ww xY w	 t          d           d| }t          j        ||          }|s*t          d           d| }t          j        ||          }	nt          j                    }	t          d           t!          |           t#          ||	|           t%          ||           t'          ||           |j        rt          dd            |                    d!d"d#d"d$d%&           |st          d'           t!          |	           t#          |	||           t%          |	|           t'          |	|           |	j        rt          d(d)           |	                    d*d"d#d"d$d%&           n# t,          $ r}
t          d+d,|
            Y d }
~
nd }
~
wt.          $ r}t          d-d.|            Y d }~nd }~wt0          $ r}t          d/d0|            Y d }~njd }~wt          j        j	        $ r*}t          d1d2t          |                      Y d }~n.d }~wt          $ r}t          d3d| d4           Y d }~nd }~ww xY w||	fS )5Nr#   Fdb_host2   z&Error: 'db_host' parameter is missing.db_user3   z&Error: 'db_user' parameter is missing.db_password4   z*Error: 'db_password' parameter is missing.db_name5   z&Error: 'db_name' parameter is missing.histViewName6   z+Error: 'histViewName' parameter is missing.futureViewName7   z-Error: 'futureViewName' parameter is missing.hostuserpassworddatabase8   z*Error: Could not connect to the database: 9   r   z" while connecting to the database.z--load histData--zSELECT * FROM z--load futureData--z--prepare histData--:   z$Error: The historical data is empty.z /var/www/prophettmp/histData.csvT;NAzutf-8)r[   sepheaderna_repencodingz--prepare futureData--;   z Error: The future data is empty.z"/var/www/prophettmp/futureData.csv<   zError: a key was not found - =   zError: Value error occurred - >   zError: Index error occurred - ?   z.Error: Could not read data from the database: @   z& while reading data from the database.)r   _dbHostr0   _dbUser_dbPassword_dbNamemysql	connectorconnectErrorr.   r   r   r   read_sql	DataFramer   r   rl   rs   emptyto_csvrM   rD   
IndexError)r6   r8   r#   r   r   connr   SQLr9   r:   r   veies                r   r2   r2   D  so    &((!"#67bBCCCYG
JJybBCCCYG
JJyF""bFGGG'K
JJ}bBCCCYG
JJyV##bGHHH.)L
JJ~ %6))"MNNN 01

#$$$
g&& 	 ' 
 
 ?  Q Q QbOs1vvOOPPPPPPPP g g gbe3q66eeeffffffffg9f!"""-|--;sD))  	('(((3>33CS$//JJJ$%%% ***+Hj*MMM+HjAAA,XzBBB> 	F"DEEE. 	 	
 	
 	
 ! 	*+++$Z000/
HjQQQ/
JGGG0ZHHH Fb"DEEE4       @ @ @b>">>???????? A A Ab?2??@@@@@@@@ A A Ab?2??@@@@@@@@?  U U UbS3q66SSTTTTTTTT f f fbd1dddeeeeeeeef Zsm   97E1 1G" F**G"7!GG"&E,M 
P&M55P&NP&'N??P& O;;P&P!!P&c           	         t          d| d           d |D             }t          d           |D ]t                     t          d           t          d           |j        D ]|vrt                     t          d           	 t          j                            t
          t          t          t                    }nY# t          j        j	        $ r}t          dd	|            Y d }~n-d }~wt          $ r}t          d
d|            Y d }~nd }~ww xY w|                                }|                                D ]\  }	}
|
d         }g }|D ]V|
         }t          j        |          rd}t!          fd|D             d           }|                    | ||||f           Wd}|d                    dgt'          |          z            z  }d |D             }	 |                    ||           # t          j        j	        $ r}t          dd|            Y d }~d }~wt          $ r}t          dd|            Y d }~
d }~ww xY w|                                 d S )Nz	--Saving z
 results--c                     g | ]
}|d          S )r   rx   )ry   items     r   r|   z'predict_saveResults.<locals>.<listcomp>  s    :::DG:::rm   zColumns to be saved:r   zColumns not saved:r   F   zDatabase connection error: H   z-Unexpected error during database connection: r   r   c              3   @   K   | ]}|d          k    |d         V  dS )r   r   Nrx   )ry   r   cols     r   	<genexpr>z&predict_saveResults.<locals>.<genexpr>  s0      VV$tAwRU~~d1g~~~~VVrm   zIINSERT INTO prediction_results (result_id, result_type, ds, k, v) VALUES z, z(%s, %s, %s, %s, %s)c                     g | ]	}|D ]}|
S rx   rx   )ry   sublistr   s      r   r|   z'predict_saveResults.<locals>.<listcomp>  s%    UUUWWUUTDUUUUrm   I   z Database query execution error: J   z)Unexpected error during query execution: )r   ru   r   r   r   r   r   r   r   r   r   r   cursoriterrowsr   r   nextr_   joinr(   executecommit)r!   result_typerb   columns_to_insertwMAPEcolumn_namesr   r   r   _rowds_valuevalues_to_insertvalue	target_idinsert_queryflattened_valuesr   s                    @r   predict_saveResultsr     s*   	
-k
-
-
-...::(9:::L	
 !!!  c



	"III	
z  l""#JJJ	"III
O&& 	 ' 
 
 ?  = = =b;;;<<<<<<<< O O ObM!MMNNNNNNNNO [[]]F ++-- O O3t9  	Y 	YCHEwu~~ VVVV2CVVVX\]]I##X{HiQV$WXXXX c		#9":SAQ=R=R"RSSS
 VU0@UUU
	O|-=>>>>$ 	F 	F 	F"DDDEEEEEEEE 	O 	O 	O"M!MMNNNNNNNN	O 	KKMMMMMsH   7C DC33D DD)H  IH,,I9IIc	                 Z   | rt          ||||||           t          t          t          f          rRP|&|D ]#}	||	                             fd          ||	<   $|&|D ]#}	||	                             fd          ||	<   $d }
|3|d         }||                                         ||                             fd                                          }|dk    r|dz  nd}||         z
  |z  |d<   d}|d                                         |k                        t                    |d	<   ||                             d
          }||                             d          }||z
  }|d|z  z
  }|d|z  z   }||         |k     ||         |k    z                      t                    |d<   d}||         	                    |d                                          |d<   ||         	                    |d                              d d          |d<   |d         dz  |d<   ||         |d         d|d         z  z   k    ||         |d         d|d         z  z
  k     z                      t                    |d<   |d	         |d         z   |d         z   dk                        t                    |d<   |||j
        s|j
        s|d         }|dk    rt          dd           t          j        |d                   |d<   t          j        |d                   |d<   t          j        |ddg         |d|g         dd          }|                    d|g          }|j
        rt          dd            |d         ||         z
  |d!<   t!          j        |d                   }|dk    r?t!          j        t!          j        |d!                             |z  }
t%          d"|
           |d!                                         }t          j        |          s|dk    rd}n||z  }t!          j        |d!                   |k                        t                    |d<   t!          j        |d         dk    t!          j        |d!                   |d         z  d#z  t           j                  |d$<   |d         j        j        |d%<   |                    d%                              d&           }|d%                             |          |d'<   t%          d(           t%          |                    d                     |d                             |                    d          d                                       d                              t                    |d<   |d                             |                    d          d$                   |d$<   |g d)                             dd*i+          }t%          d,           t%          |                    d                     |                    |dd-          }|d'         |d*         z  |d.<   |                    d%          d                                         }|                                dk    r<||z                                  |                                z  }t%          d/|           |#|d         j                            d0          |d<   |#|d         j                            d0          |d<   |
S )1Nc                 $    t          |           S Nrq   xr$   s    r   <lambda>z.predict_processAfterForecast.<locals>.<lambda>  s    c.RS>T>T rm   c                 $    t          |           S r   r   r   s    r   r   z.predict_processAfterForecast.<locals>.<lambda>	  s    N\]H^H^ rm   r   c                 (    t          | z
            S r   )abs)r   median_yhats    r   r   z.predict_processAfterForecast.<locals>.<lambda>  s    c!k/6J6J rm   g)Ǻ?r   robust_z_scorer   anomaly1g      ?g      ?g      ?anomaly2r=   )windowmin_periodsrolling_medianc                 x    t          j        t          j        | t          j        |           z
                      S r   )r3   medianr   r   s    r   r   z.predict_processAfterForecast.<locals>.<lambda>)  s'    RYrva")A,,.>'?'?@@ rm   T)rawrolling_madrolling_mad_scaledr   anomaly3anomalyc   z;Error: anomaly_threshold_n_stddev must be a positive value.r   r'   inner)onhow)r   zQError: No matching 'ds' values found between historical data and forecasted data.residualzwMAPE: d   mapedatec                     t          j        | d                   dk    rJt          j        t          j        | d                             t          j        | d                   z  dz  nt           j        S )Nr'   r   r  r  )r3   r   r   nanr   s    r   r   z.predict_processAfterForecast.<locals>.<lambda>k  sY    vaf~~"" vbfQz]3344rvaf~~EKK(* rm   
daily_mapez&First 10 rows of the common dataframe:)r   r  r'   actual_y)ru   z-First 10 rows of the common_subset dataframe:leftinterval_scaled_daily_mapezAverage Daily MAPE: r   ) predict_undoLogTransformation
isinstancerZ   floatapplyr   r   astypequantilerollingr   r   r   r   merger   r3   r   r   stdr   wherer  dtr	  groupbymapra   	set_indexfillnarenamestrftime)r%   r$   r&   forecastforecast_columnsforecast_pastforecast_past_columnsr9   histDataColumnsr   r   pred_colmad_yhatscalethreshold_zQ1Q3IQRlower_boundupper_boundwindow_sizecommontotal_ystddev	thresholddaily_mape_seriescommon_subsettempdaily_actualsaverage_daily_maper   s    `                            @r   predict_processAfterForecastr9    s     A%h0@-Qf&.	A 	A 	A .3,// `N4N' V V ( 3 34T4T4T4T U U$, ` `%23%7%=%=>^>^>^>^%_%_c""E  $A& x(//11H%++,J,J,J,JKKRRTT%-]]6!!&.x&8;&F%%O!" ()9 : > > @ @; NVVWZ[[ h((..h((..2g39n39n!)(!3k!AhxFX[fFf goopstt %-h%7%?%?{`a%?%b%b%i%i%k%k!""*8"4"<"<K]^"<"_"_U@@dUKK 	 *2-)@6)I%& h(+;"<q8L`Ca?a"aah(+;"<q8L`Ca?a"aac
&++ 	 !) 4x
7K KhWaNb bghhppqtuu 	=#<x~#<`m`s#< )+%**"[\\\ 77 n]4-@AAd (D#;/h?O1PUY_fggg sHo66 < 	s"qrrr $C[6(+;;z &%%a<< F26&"45566@E)U### 
#'')) 76?? 	<fkkII2V;I  VF:$677)CKKCPPy &+"2BF6*;M4N4NQWX[Q\4\_b4bdfdjkkv  -v #NN62288/ /
 
  &f~112CDD|6777fkk"oo $1#6#:#:6;K;KD;Q;QR[;\#]#]#d#def#g#g#n#nor#s#si  -d 3 7 78H8H8N8Nv8V W Wf
 8889@@#zIZ@[[=>>>m  $$%%%""=Tv"FF6:<6H4PZK[6[23 v..s37799 !##"3m"C!H!H!J!J]M^M^M`M`!`(*<===!$*334GHH +D14==>QRRdLrm   c                     | %|D ]"}t          j        | |                   dz
  | |<   #|%|D ]"}t          j        ||                   dz
  ||<   #|%|D ]$}t          j        ||                   dz
  ||<   #d S d S )Nr   )r3   exp)r!  r"  r#  r$  r9   r%  r   s          r   r  r    s     # 	6 	6CF8C=11A5HSMM ( 	@ 	@C!#c(:!;!;a!?M#" 	6 	6CF8C=11A5HSMM 	6 	6rm   r   )r   r   r*   pandasr   mysql.connectorr   numpyr3   r   r   r   r   rZ   r.   dictr   r;   r/   r1   rl   rs   r   r   tupler   r2   r   r9  r  rx   rm   r   <module>rA     sj   



  				            
 

 c #     6C^ C^ C^N  81 1 1,' ' 'V' ' 'V 1  1  1H!U !U !UJo E",2L,M o  o  o  o f? ? ?H] ] ]B6 6 6 6 6rm   