Issue
I am trying to fit my data using my feature selection but whenever I try I get this error
Input contains NaN, infinity or a value too large for dtype('float64').
Not sure if if there's a fix to this or if I can give it a different approach but what I am aware of is that if I am going to use a feature selection I have to apply it before my algorithm,
#!/usr/bin/env python
# coding: utf-8
# In[1]:
from sklearn.model_selection import train_test_split
# In[2]:
import pandas as pd
# In[3]:
import matplotlib.pyplot as plt
# In[4]:
import numpy as np
# In[5]:
import seaborn as sns
import statsmodels.api as sm
import mglearn as mg
get_ipython().run_line_magic('matplotlib', 'inline')
# In[6]:
from sklearn.ensemble import AdaBoostRegressor
from sklearn.feature_selection import RFE
from sklearn. feature_selection import SelectFromModel
# In[7]:
first_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv")
# In[8]:
print(first_file)
# In[9]:
second_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv")
third_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Friday-WorkingHours-Morning.pcap_ISCX.csv")
fourth_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Monday-WorkingHours.pcap_ISCX.csv")
fifth_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv")
sixth_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv")
seventh_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Tuesday-WorkingHours.pcap_ISCX.csv")
eighth_file = pd.read_csv(r"/Users/feliperivas/Downloads/MachineLearningCVE/Wednesday-workingHours.pcap_ISCX.csv")
# In[10]:
print(second_file)
# In[11]:
print(third_file)
# In[12]:
print(fourth_file)
# In[13]:
print(fifth_file)
# In[14]:
print(sixth_file)
# In[15]:
print(seventh_file)
# In[16]:
print(eighth_file)
# In[17]:
first_file = first_file.loc[:,~first_file.columns.duplicated()]
# In[18]:
print(first_file)
# In[19]:
df_list = [first_file, second_file,third_file,fourth_file, fifth_file, sixth_file, seventh_file,eighth_file]
# In[20]:
merged_data = pd.concat(df_list)
# In[21]:
print(merged_data)
# print(merged_data.shape)
# In[22]:
print(merged_data.shape)
# In[23]:
print(first_file.shape)
# In[24]:
print(second_file.shape)
# In[25]:
print(third_file.shape)
# In[26]:
print(fourth_file.shape)
# In[27]:
print(fifth_file.shape)
# In[28]:
print(sixth_file.shape)
# In[29]:
print(seventh_file.shape)
# In[30]:
print(eighth_file.shape)
# In[31]:
# 2830540 number weretrying to get
# In[32]:
# df = merged_data.dropna()
# In[33]:
print(merged_data.shape)
# In[34]:
merged_data.dropna(inplace = True)
# In[35]:
print(merged_data.shape)
# In[36]:
df = merged_data
# In[37]:
df.shape
# In[38]:
df.dropna(inplace = True)
# In[39]:
keys_num = len(df.keys())
# In[40]:
df.keys()
# In[41]:
df.head()
# In[42]:
df.keys()
# In[43]:
y= df[' Label']
# In[44]:
y
# In[45]:
X = df.drop(' Label',axis=1)
# In[46]:
X
# In[47]:
estimator = AdaBoostRegressor(n_estimators=100, random_state=0)
# In[48]:
selector = RFE(estimator, n_features_to_select=12, step=1)
# In[49]:
selector.fit(X, y)
# In[50]:
cid = AdaBoostRegressor( n_estimators=100,random_state=0)
# In[51]:
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2)
# In[52]:
cid.fit(X_train, y_train)
# In[ ]:
trainning_score = cid.score(X_train, y_train)
# In[ ]:
print("Training Score: {0}".format(trainning_score))
# In[ ]:
testing_score = cid.score(X_test, y_test)
# In[ ]:
print("Test Score: {0}".format(testing_score))
# In[ ]:
print(X_train.shape)
# In[ ]:
print(X_test.shape)
# In[ ]:
estimator = AdaBoostRegressor(n_estimators=100, random_state=0)
# In[ ]:
selector = RFE(estimator, n_features_to_select=5, step=1)
# In[ ]:
selector = selector.fit(X_train, y_train)
# In[ ]:
trainning_score = selector.score(X_train, y_train)
# In[ ]:
print("Training Score: {0}".format(trainning_score))
# In[ ]:
This is the ERROR!!!:
In[ ]: selector.fit(X, y)
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) in ----> 1 selector.fit(X, y)
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_rfe.py in fit(self, X, y) 182 The target values. 183 """ --> 184 return self._fit(X, y) 185 186 def _fit(self, X, y, step_score=None):
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_rfe.py in _fit(self, X, y, step_score) 191 192 tags = self._get_tags() --> 193 X, y = self._validate_data( 194 X, y, accept_sparse="csc", 195 ensure_min_features=2,
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/base.py in _validate_data(self, X, y, reset, validate_separately, **check_params) 431 y = check_array(y, **check_y_params) 432 else: --> 433 X, y = check_X_y(X, y, **check_params) 434 out = X, y 435
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py in inner_f(*args, **kwargs) 61 extra_args = len(args) - len(all_args) 62 if extra_args <= 0: ---> 63 return f(*args, **kwargs) 64 65 # extra_args > 0
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py in check_X_y(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator) 812 raise ValueError("y cannot be None") 813 --> 814 X = check_array(X, accept_sparse=accept_sparse, 815 accept_large_sparse=accept_large_sparse, 816 dtype=dtype, order=order, copy=copy,
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py in inner_f(*args, **kwargs) 61 extra_args = len(args) - len(all_args) 62 if extra_args <= 0: ---> 63 return f(*args, **kwargs) 64 65 # extra_args > 0
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator) 661 662 if force_all_finite: --> 663 _assert_all_finite(array, 664 allow_nan=force_all_finite == 'allow-nan') 665
~/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py in _assert_all_finite(X, allow_nan, msg_dtype) 101 not allow_nan and not np.isfinite(X).all()): 102 type_err = 'infinity' if allow_nan else 'NaN, infinity' --> 103 raise ValueError( 104 msg_err.format 105 (type_err,ValueError: Input contains NaN, infinity or a value too large for dtype('float64').
Solution
I was able to fix the problem by replacing the infinite data with nan and then dropping nan data
xy = X
xy[" Label"] = y_df
#xy = dataframe
# Replace infinite updated data with nan
X.replace([np.inf,-np. inf], np.nan, inplace=True)
# Drop rows with NaN
X. dropna (inplace=True)
Answered By - felipedrivas
0 comments:
Post a Comment
Note: Only a member of this blog may post a comment.