infra: sort imports in all .py files with isort (#2225)

This commit is contained in:
Ajay Karpur
2021-05-11 23:00:33 +00:00
committed by GitHub
parent af6667bd0b
commit 187e52fc3d
623 changed files with 3014 additions and 3048 deletions
@@ -1,27 +1,28 @@
import sys
import os
import argparse
import logging
import warnings
import time
import json
import subprocess
import copy
import json
import logging
import os
import subprocess
import sys
import time
import warnings
warnings.filterwarnings("ignore", category=FutureWarning)
import pickle
from collections import Counter
from io import StringIO
from itertools import islice
from timeit import default_timer as timer
import numpy as np
import pandas as pd
import pickle
from io import StringIO
from timeit import default_timer as timer
from itertools import islice
from collections import Counter
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
from autogluon.tabular import TabularDataset, TabularPredictor
from prettytable import PrettyTable
from autogluon.tabular import TabularPredictor, TabularDataset
def make_str_table(df):
@@ -1,23 +1,23 @@
import ast
import argparse
import logging
import warnings
import os
import json
import ast
import glob
import json
import logging
import os
import pickle
import shutil
import subprocess
import sys
import boto3
import pickle
import pandas as pd
import warnings
from collections import Counter
from timeit import default_timer as timer
import numpy as np
import seaborn as sns
import boto3
import matplotlib.pyplot as plt
import shutil
import networkx as nx
import numpy as np
import pandas as pd
import seaborn as sns
logging.basicConfig(level=logging.DEBUG)
logging.info(subprocess.call("ls -lR /opt/ml/input".split()))
@@ -29,12 +29,10 @@ from smdebug.core.writer import FileWriter
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
from prettytable import PrettyTable
import autogluon as ag
from autogluon.tabular import TabularDataset, TabularPredictor
from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS
from autogluon.tabular import TabularDataset, TabularPredictor
from prettytable import PrettyTable
# print(f'DEBUG AutoGluon version : {ag.__version__}')
@@ -74,11 +72,11 @@ def format_for_print(df):
def get_roc_auc(y_test_true, y_test_pred, labels, class_labels_internal, model_output_dir):
from sklearn.preprocessing import label_binarize
from sklearn.metrics import roc_curve, auc
from itertools import cycle
from sklearn.metrics import auc, roc_curve
from sklearn.preprocessing import label_binarize
y_test_true_binalized = label_binarize(y_test_true, classes=labels)
if len(labels) == 2:
@@ -1,13 +1,11 @@
from IPython.display import display, IFrame
from ipywidgets import interact, Image, VBox, HTML, GridspecLayout, Layout, widgets
import ipywidgets as ipyw
import boto3
import os
import tarfile
import boto3
import ipywidgets as ipyw
import pandas as pd
from IPython.display import IFrame, display
from ipywidgets import HTML, GridspecLayout, Image, Layout, VBox, interact, widgets
def search_training_jobs(job_tag_name, job_tag_value):
@@ -1,16 +1,16 @@
from __future__ import absolute_import
import os
import sys
import time
import os
from utils import ExitSignalHandler
from utils import (
write_failure_file,
print_json_object,
ExitSignalHandler,
load_json_object,
save_model_artifacts,
print_files_in_path,
print_json_object,
save_model_artifacts,
write_failure_file,
)
hyperparameters_file_path = "/opt/ml/input/config/hyperparameters.json"
@@ -1,7 +1,7 @@
import signal
import pprint
import json
import os
import pprint
import signal
from os import path
@@ -1,11 +1,11 @@
from __future__ import absolute_import
import argparse
import os
import sys
import time
import os
import argparse
from utils import save_model_artifacts, print_files_in_path
from utils import print_files_in_path, save_model_artifacts
def train(hp1, hp2, hp3, train_channel, validation_channel):
@@ -1,9 +1,8 @@
from __future__ import absolute_import
from glob import glob
import os
from os.path import basename
from os.path import splitext
from glob import glob
from os.path import basename, splitext
from setuptools import find_packages, setup
@@ -1,6 +1,7 @@
from __future__ import absolute_import
import logging
from sagemaker_training import entry_point, environment
logger = logging.getLogger(__name__)
@@ -1,11 +1,11 @@
from __future__ import absolute_import
import argparse
import os
import sys
import time
import os
import argparse
from utils import save_model_artifacts, print_files_in_path
from utils import print_files_in_path, save_model_artifacts
def train(hp1, hp2, hp3, train_channel, validation_channel):
@@ -1,11 +1,11 @@
from __future__ import absolute_import
import argparse
import os
import sys
import time
import os
import argparse
from utils import save_model_artifacts, print_files_in_path
from utils import print_files_in_path, save_model_artifacts
def train(hp1, hp2, hp3, train_channel, validation_channel):
@@ -15,15 +15,15 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import glob
import json
import os
import shutil
import signal
import socket
import subprocess
import sys
import time
import signal
import socket
import glob
def copy_files(src, dest):
@@ -1,13 +1,12 @@
import glob
import json
import os
import shutil
import signal
import socket
import subprocess
import sys
import time
import signal
import socket
import glob
from contextlib import contextmanager
@@ -1,13 +1,12 @@
import glob
import json
import os
import shutil
import signal
import socket
import subprocess
import sys
import time
import signal
import socket
import glob
from contextlib import contextmanager
@@ -1,13 +1,12 @@
import glob
import json
import os
import shutil
import signal
import socket
import subprocess
import sys
import time
import signal
import socket
import glob
from contextlib import contextmanager
@@ -1,21 +1,22 @@
import os, glob
import glob
import os
import sys
sys.path.append("/mask-rcnn-tensorflow/MaskRCNN")
from model.generalized_rcnn import ResNetFPNModel
from config import finalize_configs, config as cfg
from dataset import DetectionDataset
from tensorpack.predict.base import OfflinePredictor
from tensorpack.tfutils.sessinit import get_model_loader
from tensorpack.predict.config import PredictConfig
import numpy as np
import cv2
from itertools import groupby
from threading import Lock
import cv2
import numpy as np
from config import config as cfg
from config import finalize_configs
from dataset import DetectionDataset
from model.generalized_rcnn import ResNetFPNModel
from tensorpack.predict.base import OfflinePredictor
from tensorpack.predict.config import PredictConfig
from tensorpack.tfutils.sessinit import get_model_loader
class MaskRCNNService:
@@ -25,7 +26,7 @@ class MaskRCNNService:
# class method to load trained model and create an offline predictor
@classmethod
def get_predictor(cls):
""" load trained model"""
"""load trained model"""
with cls.lock:
# check if model is already loaded
@@ -172,11 +173,10 @@ class MaskRCNNService:
# create predictor
MaskRCNNService.get_predictor()
import json
from flask import Flask
from flask import request
from flask import Response
import base64
import json
from flask import Flask, Response, request
app = Flask(__name__)
@@ -12,12 +12,12 @@
# timeout MODEL_SERVER_TIMEOUT 70 seconds
from __future__ import print_function
import os
import signal
import subprocess
import sys
model_server_timeout = os.environ.get("MODEL_SERVER_TIMEOUT", 70)
model_server_workers = int(os.environ.get("MODEL_SERVER_WORKERS", 1))
@@ -1,22 +1,23 @@
import os, glob
import glob
import os
import sys
sys.path.append("/tensorpack/examples/FasterRCNN")
from modeling.generalized_rcnn import ResNetFPNModel
from config import finalize_configs, config as cfg
from eval import predict_image, DetectionResult
from dataset import register_coco
from tensorpack.predict.base import OfflinePredictor
from tensorpack.tfutils.sessinit import get_model_loader
from tensorpack.predict.config import PredictConfig
import numpy as np
import cv2
from itertools import groupby
from threading import Lock
import cv2
import numpy as np
from config import config as cfg
from config import finalize_configs
from dataset import register_coco
from eval import DetectionResult, predict_image
from modeling.generalized_rcnn import ResNetFPNModel
from tensorpack.predict.base import OfflinePredictor
from tensorpack.predict.config import PredictConfig
from tensorpack.tfutils.sessinit import get_model_loader
class MaskRCNNService:
@@ -26,7 +27,7 @@ class MaskRCNNService:
# class method to load trained model and create an offline predictor
@classmethod
def get_predictor(cls):
""" load trained model"""
"""load trained model"""
with cls.lock:
# check if model is already loaded
@@ -139,11 +140,10 @@ class MaskRCNNService:
# create predictor
MaskRCNNService.get_predictor()
import json
from flask import Flask
from flask import request
from flask import Response
import base64
import json
from flask import Flask, Response, request
app = Flask(__name__)
@@ -12,6 +12,7 @@
# timeout MODEL_SERVER_TIMEOUT 70 seconds
from __future__ import print_function
import os
import signal
import subprocess
@@ -1,13 +1,12 @@
import glob
import json
import os
import shutil
import signal
import socket
import subprocess
import sys
import time
import signal
import socket
import glob
from contextlib import contextmanager
@@ -15,15 +15,15 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import glob
import json
import os
import shutil
import signal
import socket
import subprocess
import sys
import time
import signal
import socket
import glob
def copy_files(src, dest):
@@ -6,17 +6,15 @@
# the root directory of this source tree. An additional grant of patent rights
# can be found in the PATENTS file in the same directory.
import json
import os
import socket
import subprocess
import json
from train_driver import main as single_process_main
from fairseq import distributed_utils, options
from multiprocessing_train import ErrorHandler
import torch
from fairseq import distributed_utils, options
from multiprocessing_train import ErrorHandler
from train_driver import main as single_process_main
def run(args, error_queue):
@@ -1,8 +1,7 @@
from sagemaker_translate import model_fn, input_fn, output_fn, predict_fn
import flask
import os
import flask
from sagemaker_translate import input_fn, model_fn, output_fn, predict_fn
prefix = "/opt/ml/"
model_path = os.path.join(prefix, "model")
@@ -9,20 +9,18 @@
Translate raw text with a trained model. Batches data on-the-fly.
"""
from collections import namedtuple
import numpy as np
import sys
import os
import logging
import json
import copy
import json
import logging
import os
import sys
from collections import namedtuple
import numpy as np
import torch
from fairseq import data, options, tasks, tokenizer, utils
from fairseq.sequence_generator import SequenceGenerator
Batch = namedtuple("Batch", "srcs tokens lengths")
Translation = namedtuple("Translation", "src_str hypos pos_scores alignments")
@@ -14,6 +14,7 @@
# timeout MODEL_SERVER_TIMEOUT 60 seconds
from __future__ import print_function
import multiprocessing
import os
import signal
@@ -1,22 +1,18 @@
#!/opt/conda/bin/python
from __future__ import print_function
import sys
import os
import copy
import json
import os
import pickle
import shutil
import sys
import traceback
import copy
import shutil
from glob import glob
from fairseq import distributed_utils, options
from train_driver import main
# These are the paths to where SageMaker mounts interesting things in your container.
prefix = '/opt/ml/'
@@ -14,14 +14,14 @@ Train a new model on one or across multiple GPUs.
import collections
import itertools
import os
import math
import torch
import os
import torch
from fairseq import distributed_utils, options, progress_bar, tasks, utils
from fairseq.data import iterators
from fairseq.trainer import Trainer
from fairseq.meters import AverageMeter, StopwatchMeter
from fairseq.trainer import Trainer
def main(args):
@@ -1,13 +1,12 @@
import argparse
import logging
import sagemaker_containers
import requests
import os
import io
import glob
import io
import logging
import os
import time
import requests
import sagemaker_containers
from fastai.vision import *
logger = logging.getLogger(__name__)
@@ -1,23 +1,19 @@
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import print_function, unicode_literals
import time
import sys
import csv
import os
import shutil
import csv
import sys
import time
import boto3
from awsglue.utils import getResolvedOptions
import pyspark
from pyspark.sql import SparkSession
from pyspark.sql.types import IntegerType, StructField, StructType, StringType
from pyspark.ml.feature import Tokenizer
from pyspark.sql.functions import *
from mleap.pyspark.spark_support import SimpleSparkSerializer
from awsglue.utils import getResolvedOptions
from mleap.pyspark.spark_support import SimpleSparkSerializer
from pyspark.ml.feature import Tokenizer
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import IntegerType, StringType, StructField, StructType
def csv_line(data):
@@ -1,22 +1,20 @@
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import print_function, unicode_literals
import time
import sys
import csv
import os
import shutil
import csv
import sys
import time
import boto3
from awsglue.utils import getResolvedOptions
import pyspark
from pyspark.sql import SparkSession
from pyspark.ml import Pipeline
from pyspark.sql.types import StructField, StructType, StringType, DoubleType
from pyspark.ml.feature import StringIndexer, VectorIndexer, OneHotEncoder, VectorAssembler
from pyspark.sql.functions import *
from awsglue.utils import getResolvedOptions
from mleap.pyspark.spark_support import SimpleSparkSerializer
from pyspark.ml import Pipeline
from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler, VectorIndexer
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import DoubleType, StringType, StructField, StructType
def csv_line(data):
@@ -1,27 +1,26 @@
from __future__ import print_function
import time
import sys
import csv
import os
import shutil
import csv
import sys
import time
import boto3
from awsglue.utils import getResolvedOptions
import pyspark
from pyspark.sql import SparkSession
from pyspark.ml import Pipeline
from pyspark.ml.feature import (
StringIndexer,
VectorIndexer,
OneHotEncoder,
VectorAssembler,
IndexToString,
)
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.sql.functions import *
from awsglue.utils import getResolvedOptions
from mleap.pyspark.spark_support import SimpleSparkSerializer
from pyspark.ml import Pipeline
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.feature import (
IndexToString,
OneHotEncoder,
StringIndexer,
VectorAssembler,
VectorIndexer,
)
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
def toCSVLine(data):
@@ -14,8 +14,8 @@
Custom Framework Estimator for JAX
"""
from sagemaker.estimator import Framework
from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
from sagemaker.tensorflow.model import TensorFlowModel
from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
class JaxEstimator(Framework):
@@ -13,9 +13,9 @@
"""
Train JAX model and serialize as TF SavedModel
"""
import time
import argparse
import functools
import time
import jax
import jax.numpy as jnp
@@ -15,8 +15,8 @@ Train Trax model and serialize as TF SavedModel
"""
import argparse
import trax
import tensorflow as tf
import trax
from trax import layers as tl
from trax.supervised import training
@@ -1,9 +1,10 @@
import os
import shlex
import subprocess
import sys
import shlex
import os
from retrying import retry
from subprocess import CalledProcessError
from retrying import retry
from sagemaker_inference import model_server
@@ -1,12 +1,12 @@
"""
ModelHandler defines an example model handler for load and inference requests for MXNet CPU models
"""
from collections import namedtuple
import glob
import json
import logging
import os
import re
from collections import namedtuple
import mxnet as mx
import numpy as np
@@ -1,23 +1,16 @@
from __future__ import print_function
import time
import sys
from io import StringIO
import os
import shutil
import argparse
import csv
import json
import os
import shutil
import sys
import time
from io import StringIO
import numpy as np
import pandas as pd
from sklearn.compose import ColumnTransformer
from sklearn.externals import joblib
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Binarizer, StandardScaler, OneHotEncoder
from sagemaker_containers.beta.framework import (
content_types,
encoders,
@@ -26,6 +19,11 @@ from sagemaker_containers.beta.framework import (
transformer,
worker,
)
from sklearn.compose import ColumnTransformer
from sklearn.externals import joblib
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Binarizer, OneHotEncoder, StandardScaler
# Since we get a headerless CSV file we specify the column names here.
feature_columns_names = [
@@ -11,7 +11,6 @@ import torch
# Local Dependencies:
from model import MNISTNet
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))
@@ -1,6 +1,5 @@
# Python Built-Ins:
import argparse
from distutils.dir_util import copy_tree
import gzip
import json
import logging
@@ -8,22 +7,22 @@ import os
import shutil
import subprocess
import sys
from distutils.dir_util import copy_tree
from tempfile import TemporaryDirectory
# External Dependencies:
import numpy as np
from packaging import version as pkgversion
from sagemaker_pytorch_serving_container import handler_service as default_handler_service
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from model import MNISTNet
from packaging import version as pkgversion
from sagemaker_pytorch_serving_container import handler_service as default_handler_service
from torch.utils.data import DataLoader, Dataset
# Local Dependencies:
from inference import *
from model import MNISTNet
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
@@ -1,11 +1,11 @@
import logging
import gzip
import mxnet as mx
import numpy as np
import logging
import os
import struct
import mxnet as mx
import numpy as np
def find_file(root_path, file_name):
for root, dirs, files in os.walk(root_path):
@@ -10,15 +10,15 @@
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import ast
import argparse
import ast
import logging
import os
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.parallel
import torch.optim
import torch.utils.data
@@ -26,7 +26,6 @@ import torch.utils.data.distributed
import torchvision
import torchvision.models
import torchvision.transforms as transforms
import torch.nn.functional as F
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
@@ -10,11 +10,11 @@
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import matplotlib.pyplot as plt
import numpy as np
import torch
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import numpy as np
classes = ("plane", "car", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck")
@@ -3,16 +3,15 @@
from __future__ import print_function
import os
import json
import pickle
import io
import sys
import json
import os
import pickle
import signal
import sys
import traceback
import flask
import pandas as pd
prefix = "/opt/ml/"
@@ -6,14 +6,13 @@
from __future__ import print_function
import os
import json
import os
import pickle
import sys
import traceback
import pandas as pd
from sklearn import tree
# These are the paths to where SageMaker mounts interesting things in your container.
@@ -10,17 +10,14 @@
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import, division, print_function
import argparse
import functools
import os
import tensorflow as tf
import resnet_model
import tensorflow as tf
INPUT_TENSOR_NAME = "inputs"
SIGNATURE_NAME = "serving_default"
@@ -29,9 +29,7 @@ The key difference of the full preactivation 'v2' variant compared to the
'v1' variant in [1] is the use of batch normalization before every weight layer
rather than after.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import, division, print_function
import tensorflow as tf
@@ -19,10 +19,10 @@
from __future__ import print_function
import os
import json
import sys
import os
import subprocess
import sys
import traceback
# These are the paths to where SageMaker mounts interesting things in your container.
@@ -16,9 +16,7 @@ Generates tf.train.Example protos and writes them to TFRecord files from the
python version of the CIFAR-10 dataset downloaded from
https://www.cs.toronto.edu/~kriz/cifar.html.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import, division, print_function
import argparse
import os
@@ -26,12 +24,12 @@ import shutil
import sys
import tarfile
from six.moves import cPickle as pickle
from six.moves import urllib
from six.moves import xrange # pylint: disable=redefined-builtin
from ipywidgets import FloatProgress
from IPython.display import display
import tensorflow as tf
from IPython.display import display
from ipywidgets import FloatProgress
from six.moves import cPickle as pickle
from six.moves import xrange # pylint: disable=redefined-builtin
from six.moves import urllib
CIFAR_FILENAME = "cifar-10-python.tar.gz"
CIFAR_DOWNLOAD_URL = "https://www.cs.toronto.edu/~kriz/" + CIFAR_FILENAME
@@ -1,4 +1,5 @@
import os
import numpy as np
import tensorflow as tf
@@ -1,12 +1,13 @@
import argparse
import numpy as np
import os
import tensorflow as tf
from mlagents_envs.environment import UnityEnvironment
import mlagents
import subprocess
import yaml
import json
import os
import subprocess
import mlagents
import numpy as np
import tensorflow as tf
import yaml
from mlagents_envs.environment import UnityEnvironment
def parse_args():
@@ -1,23 +1,16 @@
from __future__ import print_function
import time
import sys
from io import StringIO
import os
import shutil
import argparse
import csv
import json
import os
import shutil
import sys
import time
from io import StringIO
import numpy as np
import pandas as pd
from sklearn.externals import joblib
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.svm import SVR
from sklearn.feature_selection import f_regression, mutual_info_regression, SelectKBest, RFE
from sagemaker_containers.beta.framework import (
content_types,
encoders,
@@ -26,6 +19,11 @@ from sagemaker_containers.beta.framework import (
transformer,
worker,
)
from sklearn.externals import joblib
from sklearn.feature_selection import RFE, SelectKBest, f_regression, mutual_info_regression
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.svm import SVR
label_column = "y"
INPUT_FEATURES_SIZE = 100
@@ -3,17 +3,16 @@
from __future__ import print_function
import os
import json
import os
import pickle
import StringIO
import sys
import signal
import sys
import traceback
import flask
import pandas as pd
import StringIO
prefix = "/opt/ml/"
model_path = os.path.join(prefix, "model")
@@ -14,6 +14,7 @@
# timeout MODEL_SERVER_TIMEOUT 60 seconds
from __future__ import print_function
import multiprocessing
import os
import signal
@@ -6,17 +6,16 @@
from __future__ import print_function
import os
import json
import os
import pickle
import sys
import traceback
import pandas as pd
from numpy import mean
from sklearn import tree
from sklearn.model_selection import cross_val_score
from numpy import mean
# These are the paths to where SageMaker mounts interesting things in your container.
@@ -10,9 +10,10 @@
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
import boto3
from datetime import datetime, timedelta
import re
from datetime import datetime, timedelta
import boto3
client = boto3.client("sagemaker")
running_jobs = client.list_training_jobs(CreationTimeAfter=datetime.utcnow() - timedelta(hours=1))
@@ -15,24 +15,24 @@ from __future__ import absolute_import, division, print_function
import argparse
import json
import logging
import re
import os
import re
import keras
import tensorflow as tf
from keras import backend as K
from keras.callbacks import TensorBoard, ModelCheckpoint
from keras.callbacks import ModelCheckpoint, TensorBoard
from keras.layers import (
Activation,
BatchNormalization,
Conv2D,
Dense,
Dropout,
Flatten,
MaxPooling2D,
BatchNormalization,
)
from keras.models import Sequential
from keras.optimizers import Adam, SGD, RMSprop
from keras.optimizers import SGD, Adam, RMSprop
logging.getLogger().setLevel(logging.INFO)
tf.logging.set_verbosity(tf.logging.INFO)
@@ -12,8 +12,8 @@ from itertools import chain, islice
import mxnet as mx
import numpy as np
from mxnet import gluon, autograd, nd
from mxnet.io import DataIter, DataBatch, DataDesc
from mxnet import autograd, gluon, nd
from mxnet.io import DataBatch, DataDesc, DataIter
logging.basicConfig(level=logging.DEBUG)
@@ -2,6 +2,7 @@ from __future__ import print_function
import json
import logging
import mxnet as mx
import mxnet.contrib.onnx as onnx_mxnet
import numpy as np
@@ -1,6 +1,7 @@
import json
import random
# sample preprocess_handler (to be implemented by customer)
# This is a trivial example, where we demonstrate an echo preprocessor for json data
# for others though, we are generating random data (real customers would not do that obviously/hopefully)
@@ -16,14 +16,13 @@ import argparse
import json
import logging
import os
import pandas as pd
import pickle as pkl
from sagemaker_containers import entry_point
from sagemaker_xgboost_container.data_utils import get_dmatrix
from sagemaker_xgboost_container import distributed
import pandas as pd
import xgboost as xgb
from sagemaker_containers import entry_point
from sagemaker_xgboost_container import distributed
from sagemaker_xgboost_container.data_utils import get_dmatrix
def _xgb_train(params, dtrain, evals, num_boost_round, model_dir, is_master):
@@ -15,12 +15,11 @@ from __future__ import print_function
import argparse
import os
import pandas as pd
import pandas as pd
from sklearn import tree
from sklearn.externals import joblib
if __name__ == "__main__":
parser = argparse.ArgumentParser()
@@ -1,5 +1,7 @@
# Standard Library
import argparse
import os
import time
# Third Party
import mxnet as mx
@@ -7,8 +9,6 @@ import numpy as np
from mxnet import autograd, gluon, init
from mxnet.gluon import nn
from mxnet.gluon.data.vision import datasets, transforms
import os
import time
def parse_args():
@@ -1,6 +1,7 @@
import numpy as np
import math
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.animation import FuncAnimation
plt.rcParams.update({"font.size": 8})
@@ -11,14 +11,13 @@ For more information, please refer to https://github.com/awslabs/sagemaker-debug
# Standard Library
import argparse
import logging
import random
# Third Party
import numpy as np
import tensorflow as tf
import smdebug.tensorflow as smd
import logging
import tensorflow as tf
logging.getLogger().setLevel(logging.INFO)
@@ -8,14 +8,13 @@ For more information, please refer to https://github.com/awslabs/sagemaker-debug
# Standard Library
import argparse
import logging
import random
# Third Party
import numpy as np
import tensorflow as tf
import logging
logging.getLogger().setLevel(logging.INFO)
parser = argparse.ArgumentParser()
@@ -1,4 +1,5 @@
import os
import tensorflow as tf
from tensorflow.python.estimator.model_fn import ModeKeys as Modes
@@ -110,9 +111,10 @@ def _input_fn(training_dir, training_filename, batch_size=100):
def neo_preprocess(payload, content_type):
import logging
import numpy as np
import io
import logging
import numpy as np
logging.info("Invoking user-defined pre-processing function")
@@ -132,9 +134,10 @@ def neo_preprocess(payload, content_type):
### NOTE: this function cannot use MXNet
def neo_postprocess(result):
import logging
import numpy as np
import json
import logging
import numpy as np
logging.info("Invoking user-defined post-processing function")
@@ -1,5 +1,6 @@
"""Converts MNIST data to TFRecords file format with Example protos."""
import os
import tensorflow as tf
@@ -1,9 +1,10 @@
import io
import os
import PIL.Image
import json
import logging
import os
import numpy as np
import PIL.Image
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
@@ -12,14 +13,13 @@ logger.setLevel(logging.DEBUG)
# Training methods #
# ------------------------------------------------------------ #
import argparse
import glob
import time
import argparse
import warnings
import mxnet as mx
from mxnet import nd
from mxnet import gluon
from mxnet import autograd
from mxnet import autograd, gluon, nd
def parse_args():
@@ -74,7 +74,7 @@ def get_dataloader(net, data_shape, batch_size, num_workers, ctx):
"""Get dataloader."""
from gluoncv import data as gdata
from gluoncv.data.batchify import Tuple, Stack, Pad
from gluoncv.data.batchify import Pad, Stack, Tuple
from gluoncv.data.transforms.presets.ssd import SSDDefaultTrainTransform
width, height = data_shape, data_shape
@@ -15,9 +15,10 @@
# specific language governing permissions and limitations
# under the License.
from imdb import Imdb
import random
from imdb import Imdb
class ConcatDB(Imdb):
"""
@@ -18,18 +18,20 @@
# under the License.
from __future__ import print_function
import os
import sys
curr_path = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(curr_path, "../python"))
import mxnet as mx
import random
import argparse
import cv2
import random
import time
import traceback
import cv2
import mxnet as mx
try:
import multiprocessing
except ImportError:
@@ -15,9 +15,10 @@
# specific language governing permissions and limitations
# under the License.
import numpy as np
import os.path as osp
import numpy as np
class Imdb(object):
"""
@@ -16,11 +16,13 @@
# under the License.
from __future__ import print_function
import os
import xml.etree.ElementTree as ET
import cv2
import numpy as np
from imdb import Imdb
import xml.etree.ElementTree as ET
import cv2
class PascalVoc(Imdb):
@@ -19,14 +19,16 @@
# under the License.
from __future__ import print_function
import sys, os
import argparse
import os
import subprocess
import sys
curr_path = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(curr_path, ".."))
from pascal_voc import PascalVoc
from concat_db import ConcatDB
from pascal_voc import PascalVoc
def load_pascal(image_set, year, devkit_path, shuffle=False):
@@ -1,12 +1,13 @@
import numpy as np
import io
import json
import logging
import os
import mxnet as mx
# Please make sure to import neomx
import neomx # noqa: F401
import io
import os
import logging
import numpy as np
# Change the context to mx.gpu() if deploying to a GPU endpoint
ctx = mx.cpu()
@@ -4,9 +4,9 @@ import logging
import os
import pickle
import neopytorch
import numpy as np
import torch
import neopytorch
import torchvision.transforms as transforms
from PIL import Image # Training container doesn't have this package
@@ -4,9 +4,9 @@ import logging
import os
import pickle
import neopytorch
import numpy as np
import torch
import neopytorch
import torchvision.transforms as transforms
from PIL import Image # Training container doesn't have this package
@@ -4,9 +4,9 @@ import logging
import os
import pickle
import neopytorch
import numpy as np
import torch
import neopytorch
import torchvision.transforms as transforms
from PIL import Image # Training container doesn't have this package
@@ -1,4 +1,5 @@
import os
import tensorflow as tf
from tensorflow.python.estimator.model_fn import ModeKeys as Modes
@@ -111,9 +112,10 @@ def _input_fn(training_dir, training_filename, batch_size=100):
def neo_preprocess(payload, content_type):
import logging
import numpy as np
import io
import logging
import numpy as np
logging.info("Invoking user-defined pre-processing function")
@@ -133,9 +135,10 @@ def neo_preprocess(payload, content_type):
### NOTE: this function cannot use MXNet
def neo_postprocess(result):
import logging
import numpy as np
import json
import logging
import numpy as np
logging.info("Invoking user-defined post-processing function")
@@ -1,5 +1,6 @@
"""Converts MNIST data to TFRecords file format with Example protos."""
import os
import tensorflow as tf
@@ -10,17 +10,14 @@
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import, division, print_function
import argparse
import functools
import os
import tensorflow as tf
import resnet_model
import tensorflow as tf
INPUT_TENSOR_NAME = "inputs"
SIGNATURE_NAME = "serving_default"
@@ -29,9 +29,7 @@ The key difference of the full preactivation 'v2' variant compared to the
'v1' variant in [1] is the use of batch normalization before every weight layer
rather than after.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import, division, print_function
import tensorflow as tf
@@ -19,10 +19,10 @@
from __future__ import print_function
import os
import json
import sys
import os
import subprocess
import sys
import traceback
# These are the paths to where SageMaker mounts interesting things in your container.
@@ -16,9 +16,7 @@ Generates tf.train.Example protos and writes them to TFRecord files from the
python version of the CIFAR-10 dataset downloaded from
https://www.cs.toronto.edu/~kriz/cifar.html.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import, division, print_function
import argparse
import os
@@ -26,12 +24,12 @@ import shutil
import sys
import tarfile
from six.moves import cPickle as pickle
from six.moves import urllib
from six.moves import xrange # pylint: disable=redefined-builtin
from ipywidgets import FloatProgress
from IPython.display import display
import tensorflow as tf
from IPython.display import display
from ipywidgets import FloatProgress
from six.moves import cPickle as pickle
from six.moves import xrange # pylint: disable=redefined-builtin
from six.moves import urllib
CIFAR_FILENAME = "cifar-10-python.tar.gz"
CIFAR_DOWNLOAD_URL = "https://www.cs.toronto.edu/~kriz/" + CIFAR_FILENAME
@@ -3,16 +3,15 @@
from __future__ import print_function
import os
import json
import os
import pickle
from io import StringIO
import sys
import signal
import sys
import traceback
from io import StringIO
import flask
import pandas as pd
import xgboost
@@ -14,6 +14,7 @@
# timeout MODEL_SERVER_TIMEOUT 60 seconds
from __future__ import print_function
import multiprocessing
import os
import signal
@@ -6,15 +6,15 @@
from __future__ import print_function
import os
import json
import os
import pickle
import sys
import traceback
import pandas as pd
import xgboost as xgb
import smdebug.xgboost as smd
import xgboost as xgb
print("Libraries imported")
@@ -1,44 +1,26 @@
from __future__ import absolute_import
from __future__ import print_function
from string import Template
import sys
import time
import os
import multiprocessing
import signal
import subprocess
from urllib.parse import urlparse
from utils import ExitSignalHandler
from utils import (
write_failure_file,
print_json_object,
load_json_object,
save_model_artifacts,
print_files_in_path,
)
import traceback
from io import StringIO
import os
import shutil
from __future__ import absolute_import, print_function
import argparse
import csv
import decimal
import json
import numpy as np
import pandas as pd
from joblib import dump, load
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import LabelEncoder
import multiprocessing
import os
import shutil
import signal
import subprocess
import sys
import time
import traceback
from io import StringIO
from string import Template
from urllib.parse import urlparse
import boto3
import decimal
import numpy as np
import pandas as pd
from botocore.exceptions import ClientError
from joblib import dump, load
from sagemaker_containers.beta.framework import (
content_types,
encoders,
@@ -47,6 +29,17 @@ from sagemaker_containers.beta.framework import (
transformer,
worker,
)
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import LabelEncoder
from utils import (
ExitSignalHandler,
load_json_object,
print_files_in_path,
print_json_object,
save_model_artifacts,
write_failure_file,
)
cpu_count = multiprocessing.cpu_count()
@@ -3,21 +3,22 @@
from __future__ import print_function
import os
import sys
import stat
import json
import shutil
import flask
from flask import Flask, jsonify, request, Response
import glob
import pandas as pd
import numpy as np
import random
import csv
import glob
import json
import os
import random
import shutil
import stat
import sys
from io import StringIO
from joblib import dump, load
import boto3
import flask
import numpy as np
import pandas as pd
from flask import Flask, Response, jsonify, request
from joblib import dump, load
from sagemaker_containers.beta.framework import (
content_types,
encoders,
@@ -1,7 +1,7 @@
import signal
import pprint
import json
import os
import pprint
import signal
from os import path
@@ -1,44 +1,25 @@
from __future__ import absolute_import
from __future__ import print_function
from string import Template
import sys
import time
import os
import shutil
import multiprocessing
import signal
import subprocess
from utils import ExitSignalHandler
from utils import (
write_failure_file,
print_json_object,
load_json_object,
save_model_artifacts,
print_files_in_path,
)
import traceback
from io import StringIO
import os
from __future__ import absolute_import, print_function
import argparse
import csv
import decimal
import json
import numpy as np
import pandas as pd
from joblib import dump, load
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import LabelEncoder
import multiprocessing
import os
import shutil
import signal
import subprocess
import sys
import time
import traceback
from io import StringIO
from string import Template
import boto3
import decimal
import numpy as np
import pandas as pd
from botocore.exceptions import ClientError
from joblib import dump, load
from sagemaker_containers.beta.framework import (
content_types,
encoders,
@@ -47,6 +28,17 @@ from sagemaker_containers.beta.framework import (
transformer,
worker,
)
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import LabelEncoder
from utils import (
ExitSignalHandler,
load_json_object,
print_files_in_path,
print_json_object,
save_model_artifacts,
write_failure_file,
)
hyperparameters_file_path = "/opt/ml/input/config/hyperparameters.json"
inputdataconfig_file_path = "/opt/ml/input/config/inputdataconfig.json"
@@ -3,18 +3,19 @@
from __future__ import print_function
import os
import sys
import stat
import json
import shutil
import flask
from flask import Flask, jsonify, request, make_response, Response
import glob
import pandas as pd
import numpy as np
import csv
import glob
import json
import os
import shutil
import stat
import sys
from io import StringIO
import flask
import numpy as np
import pandas as pd
from flask import Flask, Response, jsonify, make_response, request
from joblib import dump, load
from sagemaker_containers.beta.framework import (
content_types,
@@ -24,12 +25,13 @@ from sagemaker_containers.beta.framework import (
transformer,
worker,
)
from utils import (
write_failure_file,
print_json_object,
load_json_object,
save_model_artifacts,
print_files_in_path,
print_json_object,
save_model_artifacts,
write_failure_file,
)
model_artifacts_path = "/opt/ml/model/"
@@ -1,7 +1,7 @@
import signal
import pprint
import json
import os
import pprint
import signal
from os import path
@@ -1,6 +1,6 @@
import argparse
import random
import csv
import random
parser = argparse.ArgumentParser(description="Generate sample data")
parser.add_argument("--samples", type=int, default=10000, help="Number of samples to generate")
@@ -1,5 +1,6 @@
import argparse
import csv
import boto3
parser = argparse.ArgumentParser(description="Load DynamoDB data")
+4 -3
View File
@@ -1,8 +1,9 @@
import time
import boto3
import argparse
import pandas as pd
import pathlib
import time
import boto3
import pandas as pd
# Parse argument variables passed via the CreateDataset processing step
parser = argparse.ArgumentParser()
+2 -1
View File
@@ -1,6 +1,7 @@
import boto3
import time
import boto3
def delete_project_resources(
sagemaker_boto_client,
+2 -2
View File
@@ -1,7 +1,7 @@
import time
import boto3
import argparse
import time
import boto3
# Parse argument variables passed via the DeployModel processing step
parser = argparse.ArgumentParser()
+6 -5
View File
@@ -1,10 +1,11 @@
import os
import sys
import pickle
import xgboost as xgb
import argparse
import pandas as pd
import json
import os
import pickle
import sys
import pandas as pd
import xgboost as xgb
if __name__ == "__main__":
parser = argparse.ArgumentParser()
+5 -6
View File
@@ -15,15 +15,14 @@
from __future__ import print_function
import logging
from mxnet import gluon
import mxnet as mx
import numpy as np
import json
import logging
import os
import mxnet as mx
import numpy as np
from mxnet import gluon
logging.basicConfig(level=logging.DEBUG)
+5 -3
View File
@@ -13,16 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from inference import transform_fn, model_fn
import os
import json
import os
import shutil
import tarfile
import boto3
import botocore
import tarfile
import numpy as np
import sagemaker
from inference import model_fn, transform_fn
def fetch_model(model_data):
"""Untar the model.tar.gz object either from local file system
+5 -5
View File
@@ -13,12 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from train import train, parse_args
import sys
import os
import boto3
import json
import os
import sys
import boto3
from train import parse_args, train
dirname = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(dirname, "config.json"), "r") as f:

Some files were not shown because too many files have changed in this diff Show More