LendingClub数据的分析建模(一)

pip install -r requirements.txt

# coding:utf-8
import sys
import pandas as pd
import numpy as np
import os,time
from pathlib import Path
from datetime import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import ofun.o_utils as o_utils
import math
import reportgen as rpt
from wordcloud import WordCloud
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.preprocessing   import MinMaxScaler
from sklearn.cluster import  KMeans
from sklearn.linear_model import  LogisticRegression
from sklearn.metrics import roc_auc_score,roc_curve
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV


from statsmodels.stats.outliers_influence  import variance_inflation_factor as vif
import statsmodels.api as sm
from scipy.stats  import mode


plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号,#有中文出现的情况，需要u'内容'
# plt.rcParams.update({'figure.max_open_warning': 0})
# 显示设置
np.set_printoptions(threshold=sys.maxsize)  # 这个参数填的是你想要多少行显示
np.set_printoptions(linewidth=50000)  # 这个参数填的是横向多宽
pd.set_option('display.max_columns', 50000)
pd.set_option('display.max_rows', 50000)
pd.set_option('display.width', 50000)  # 影响显示
pd.set_option('display.max_colwidth', 50000)
start_time = time.process_time()




# stage = ['loadFile','dataanyla','dataFilter','dataClrAndPre','genFeatures']   # dataPPT (可选)
stage = ['loadFile','dataanyla','dataFilter','dataClrAndPre']
print("一：课题分析：构造一个简单明了的传统信贷申请评分卡（Ａ卡），和一个解释性较差的黑箱预测模型，用于辅助决策")


disk_file='F:/jcbank青干班/D11/34_1110834_bundle_archive/LCDataDictionary.xlsx'
disk = pd.read_excel(disk_file,sheet_name='LoanStats')
data_disc = dict(zip(disk['LoanStatNew'],disk['LoanStatNew_trans']))


data = None
file = "lc_2016-2018.pkl"
file_test = "lc_test_2016-2018.pkl"
file_vail = "lc_vail_2016-2018.pkl"
file_train = "lc_train_2016-2018.pkl"
df_list = []
if(Path(file).exists()):
    df = pd.read_pickle(file)
    df_list.append(df)
if (Path(file_test).exists()):
    df_test = pd.read_pickle(file_test)
    df_list.append(df_test)
if (Path(file_vail).exists()):
    df_vail = pd.read_pickle(file_vail)
    df_list.append(df_vail)
if (Path(file_train).exists()):
    df_train = pd.read_pickle(file_train)
    df_list.append(df_train)
    
    # df_list = [df, df_test, df_vail, df_train]
    print("loadFile" + str(df_train.shape))

'''
  二：数据获取
'''
# 根据贷款状态 ‘loan_status’ 定义好坏贷款
bad_loan = ["Charged Off",
                "Default",
                "Does not meet the credit policy. Status:Charged Off",
                "In Grace Period",
                "Late (16-30 days)",
                "Late (31-120 days)"]
def loan_condition(status):
    if status in bad_loan:
        return 'Bad Loan'
    else:
        return 'Good Loan'
if 'loadFile' in stage:
    print("二：数据获取:")
    afile='D:/workspace/python3/OupFlask/apps/lc/data/loan.csv'
    data = pd.read_csv(afile,low_memory=False)
    print(data.info())


    print(o_utils.get_bp(2) + "1)、数据分拆: issue_d(2016-2018年)")
    # print(df['issue_d'].value_counts)   #过滤字段  值初评估
    dt_series = pd.to_datetime(data['issue_d'])  # 将 ‘issue_d’ str形式的时间日期转换
    data['year'] = dt_series.dt.year
    data['month'] = dt_series.dt.month
    df = data[(data[u'year'] >= 2016) & (data[u'year'] <= 2018)]  # 过滤
    df.loc[:, 'loan_condition'] = np.nan  # 初始化字段
    df.loc[:, 'loan_condition'] = df['loan_status'].apply(loan_condition)
    df.loc[:, 'loan_condition_int'] = np.nan  # 初始化字段
    lst = [df]
    for col in lst:
        col.loc[df['loan_condition'] == 'Good Loan', 'loan_condition_int'] = 0  # Negative (Bad Loan)
        col.loc[df['loan_condition'] == 'Bad Loan', 'loan_condition_int'] = 1  # Positive (Good Loan)
    df.loc[:, 'loan_condition_int'] = df['loan_condition_int'].astype(int)


    print(df.info())
    df.to_pickle(file)


    # badloanDf = df[df['loan_condition'] == 'Bad Loan']  # 整理出坏账数据