基于AutoML的航班延迟起飞预测模型搭建

import pandas as pd


import warnings
warnings.filterwarnings('ignore')


import logging
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.ERROR)


from ads.dataset.factory import DatasetFactory

flights = DatasetFactory.open("/home/datascience/ads-examples/3P_data/flights.csv")flights.head()

flights = flights.set_target('DepDelay')

flights.show_in_notebook()

flights_nocheat = flights.drop_columns(["ArrDelay", "ArrTime", "DepTime", 
                                            "AirTime", "ActualElapsedTime"]).sample(frac=0.1)

flights_nocheat.target.show_in_notebook(feature_names=["DayOfWeek", "DayofMonth", 
                                                           "UniqueCarrier"])

type(flights_nocheat)
# RegressionDataset

# 转换前show_in_notebook() 默认显示为 Histogram Graph
flights_nocheat.target.show_in_notebook()

flights_binary = flights_nocheat.assign_column("DepDelay", lambda x:  x is None or x>15)
type(flights_binary)
# BinaryClassificationDataset

# 数据转换后, show_in_notebook() 默认显示为Count Plot Graph
flights_binary.target.show_in_notebook()

flights_binary.get_recommendations()

flights_clean = flights_binary.auto_transform(fix_imbalance=False)

flights_clean.show_in_notebook()

[len(flights_binary.columns), len(flights_clean.columns)]

# 相关性Top12变量选择
modeling_dataset = flights_clean.select_best_features(k=12)
print(modeling_dataset.columns.values)

import h2o
import os
from h2o.automl import H2OAutoML

# H2O Proxy 初始化
http_proxy = os.environ.get("http_proxy", "")
HTTP_PROXY = os.environ.get("HTTP_PROXY", "")
os.environ["HTTP_PROXY"] = ""
os.environ["http_proxy"] = ""
h2o.init()

h2o_df = modeling_dataset.to_h2o_dataframe()
h2o_df.head(5)

train, valid, test = h2o_df.split_frame(ratios=list([.7, .15]), seed = 1)
[train.shape, valid.shape, test.shape]

x = train.columns
y = "DepDelay"
x.remove(y)

%%time
h2o_aml = H2OAutoML(max_models=20, seed=1, max_runtime_secs=100)
h2o_aml.train(x=x, y=y, training_frame=train, validation_frame=valid)
h2o_model = h2o_aml.leader

print("\nH2O Accuracy: ", (h2o_model.predict(test)[0] == test["DepDelay"]).sum() / test.shape[0])

os.environ["http_proxy"] = http_proxy
os.environ["HTTP_PROXY"] = HTTP_PROXY

from ads.automl.provider import OracleAutoMLProvider

train, valid, test = modeling_dataset.train_validation_test_split(test_size=0.1, validation_size=0.2, random_state=42)
[train.X.shape, valid.X.shape, test.X.shape]

olabs_automl = OracleAutoMLProvider()

import lightgbm

%%time
from ads.automl.driver import AutoML
automl = AutoML(train, validation_data=valid, 
                provider=olabs_automl)
olabs_model, baseline = automl.train(min_features=["CRSDepTime", "TaxiOut"], model_list=["LogisticRegression", "LGBMClassifier", "XGBClassifier", "RandomForestClassifier"], random_state = 42, score_metric = 'roc_auc')

print("\nOLabs Score: ", olabs_model.score(test.X, test.y))

olabs_model.selected_model_params_

olabs_model.num_total_trials_

olabs_model.summary()

automl.visualize_feature_selection_trials()

automl.visualize_tuning_trials()

olabs_model.visualize_transforms()

from ads.evaluations.evaluator import ADSEvaluator
from ads.common.data import MLData


evaluator = ADSEvaluator(test, models=[baseline, olabs_model])
evaluator.show_in_notebook()

from ads.explanations.explainer import ADSExplainer
from ads.explanations.mlx_global_explainer import MLXGlobalExplainer

olabs_explainer = ADSExplainer(test, olabs_model)
global_explainer = olabs_explainer.global_explanation(provider=MLXGlobalExplainer())

global_explainer.feature_importance_summary()

feature_importance = global_explainer.compute_feature_importance()

feature_importance.get_global_explanation()