infra: sort imports in all .py files with isort (#2225)
This commit is contained in:
@@ -1,27 +1,28 @@
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import warnings
|
||||
import time
|
||||
import json
|
||||
import subprocess
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore", category=FutureWarning)
|
||||
|
||||
import pickle
|
||||
from collections import Counter
|
||||
from io import StringIO
|
||||
from itertools import islice
|
||||
from timeit import default_timer as timer
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pickle
|
||||
from io import StringIO
|
||||
from timeit import default_timer as timer
|
||||
from itertools import islice
|
||||
from collections import Counter
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
from autogluon.tabular import TabularDataset, TabularPredictor
|
||||
from prettytable import PrettyTable
|
||||
from autogluon.tabular import TabularPredictor, TabularDataset
|
||||
|
||||
|
||||
def make_str_table(df):
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
import ast
|
||||
import argparse
|
||||
import logging
|
||||
import warnings
|
||||
import os
|
||||
import json
|
||||
import ast
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pickle
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import boto3
|
||||
import pickle
|
||||
import pandas as pd
|
||||
import warnings
|
||||
from collections import Counter
|
||||
from timeit import default_timer as timer
|
||||
|
||||
import numpy as np
|
||||
import seaborn as sns
|
||||
import boto3
|
||||
import matplotlib.pyplot as plt
|
||||
import shutil
|
||||
import networkx as nx
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import seaborn as sns
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
logging.info(subprocess.call("ls -lR /opt/ml/input".split()))
|
||||
@@ -29,12 +29,10 @@ from smdebug.core.writer import FileWriter
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
from prettytable import PrettyTable
|
||||
import autogluon as ag
|
||||
|
||||
from autogluon.tabular import TabularDataset, TabularPredictor
|
||||
|
||||
from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS
|
||||
from autogluon.tabular import TabularDataset, TabularPredictor
|
||||
from prettytable import PrettyTable
|
||||
|
||||
# print(f'DEBUG AutoGluon version : {ag.__version__}')
|
||||
|
||||
@@ -74,11 +72,11 @@ def format_for_print(df):
|
||||
|
||||
|
||||
def get_roc_auc(y_test_true, y_test_pred, labels, class_labels_internal, model_output_dir):
|
||||
from sklearn.preprocessing import label_binarize
|
||||
from sklearn.metrics import roc_curve, auc
|
||||
|
||||
from itertools import cycle
|
||||
|
||||
from sklearn.metrics import auc, roc_curve
|
||||
from sklearn.preprocessing import label_binarize
|
||||
|
||||
y_test_true_binalized = label_binarize(y_test_true, classes=labels)
|
||||
|
||||
if len(labels) == 2:
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
from IPython.display import display, IFrame
|
||||
from ipywidgets import interact, Image, VBox, HTML, GridspecLayout, Layout, widgets
|
||||
import ipywidgets as ipyw
|
||||
|
||||
import boto3
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
|
||||
import boto3
|
||||
import ipywidgets as ipyw
|
||||
import pandas as pd
|
||||
from IPython.display import IFrame, display
|
||||
from ipywidgets import HTML, GridspecLayout, Image, Layout, VBox, interact, widgets
|
||||
|
||||
|
||||
def search_training_jobs(job_tag_name, job_tag_value):
|
||||
|
||||
+5
-5
@@ -1,16 +1,16 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
from utils import ExitSignalHandler
|
||||
from utils import (
|
||||
write_failure_file,
|
||||
print_json_object,
|
||||
ExitSignalHandler,
|
||||
load_json_object,
|
||||
save_model_artifacts,
|
||||
print_files_in_path,
|
||||
print_json_object,
|
||||
save_model_artifacts,
|
||||
write_failure_file,
|
||||
)
|
||||
|
||||
hyperparameters_file_path = "/opt/ml/input/config/hyperparameters.json"
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import signal
|
||||
import pprint
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
import signal
|
||||
from os import path
|
||||
|
||||
|
||||
|
||||
+3
-3
@@ -1,11 +1,11 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
import argparse
|
||||
|
||||
from utils import save_model_artifacts, print_files_in_path
|
||||
from utils import print_files_in_path, save_model_artifacts
|
||||
|
||||
|
||||
def train(hp1, hp2, hp3, train_channel, validation_channel):
|
||||
|
||||
+2
-3
@@ -1,9 +1,8 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
from glob import glob
|
||||
import os
|
||||
from os.path import basename
|
||||
from os.path import splitext
|
||||
from glob import glob
|
||||
from os.path import basename, splitext
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
|
||||
+1
@@ -1,6 +1,7 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import logging
|
||||
|
||||
from sagemaker_training import entry_point, environment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+3
-3
@@ -1,11 +1,11 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
import argparse
|
||||
|
||||
from utils import save_model_artifacts, print_files_in_path
|
||||
from utils import print_files_in_path, save_model_artifacts
|
||||
|
||||
|
||||
def train(hp1, hp2, hp3, train_channel, validation_channel):
|
||||
|
||||
+3
-3
@@ -1,11 +1,11 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
import argparse
|
||||
|
||||
from utils import save_model_artifacts, print_files_in_path
|
||||
from utils import print_files_in_path, save_model_artifacts
|
||||
|
||||
|
||||
def train(hp1, hp2, hp3, train_channel, validation_channel):
|
||||
|
||||
@@ -15,15 +15,15 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import glob
|
||||
|
||||
|
||||
def copy_files(src, dest):
|
||||
|
||||
+3
-4
@@ -1,13 +1,12 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import glob
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
|
||||
+3
-4
@@ -1,13 +1,12 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import glob
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
|
||||
+3
-4
@@ -1,13 +1,12 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import glob
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
|
||||
+16
-16
@@ -1,21 +1,22 @@
|
||||
import os, glob
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append("/mask-rcnn-tensorflow/MaskRCNN")
|
||||
|
||||
from model.generalized_rcnn import ResNetFPNModel
|
||||
from config import finalize_configs, config as cfg
|
||||
from dataset import DetectionDataset
|
||||
|
||||
from tensorpack.predict.base import OfflinePredictor
|
||||
from tensorpack.tfutils.sessinit import get_model_loader
|
||||
from tensorpack.predict.config import PredictConfig
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
from itertools import groupby
|
||||
from threading import Lock
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from config import config as cfg
|
||||
from config import finalize_configs
|
||||
from dataset import DetectionDataset
|
||||
from model.generalized_rcnn import ResNetFPNModel
|
||||
from tensorpack.predict.base import OfflinePredictor
|
||||
from tensorpack.predict.config import PredictConfig
|
||||
from tensorpack.tfutils.sessinit import get_model_loader
|
||||
|
||||
|
||||
class MaskRCNNService:
|
||||
|
||||
@@ -25,7 +26,7 @@ class MaskRCNNService:
|
||||
# class method to load trained model and create an offline predictor
|
||||
@classmethod
|
||||
def get_predictor(cls):
|
||||
""" load trained model"""
|
||||
"""load trained model"""
|
||||
|
||||
with cls.lock:
|
||||
# check if model is already loaded
|
||||
@@ -172,11 +173,10 @@ class MaskRCNNService:
|
||||
# create predictor
|
||||
MaskRCNNService.get_predictor()
|
||||
|
||||
import json
|
||||
from flask import Flask
|
||||
from flask import request
|
||||
from flask import Response
|
||||
import base64
|
||||
import json
|
||||
|
||||
from flask import Flask, Response, request
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
+1
-1
@@ -12,12 +12,12 @@
|
||||
# timeout MODEL_SERVER_TIMEOUT 70 seconds
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
model_server_timeout = os.environ.get("MODEL_SERVER_TIMEOUT", 70)
|
||||
model_server_workers = int(os.environ.get("MODEL_SERVER_WORKERS", 1))
|
||||
|
||||
|
||||
+17
-17
@@ -1,22 +1,23 @@
|
||||
import os, glob
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append("/tensorpack/examples/FasterRCNN")
|
||||
|
||||
from modeling.generalized_rcnn import ResNetFPNModel
|
||||
from config import finalize_configs, config as cfg
|
||||
from eval import predict_image, DetectionResult
|
||||
from dataset import register_coco
|
||||
|
||||
from tensorpack.predict.base import OfflinePredictor
|
||||
from tensorpack.tfutils.sessinit import get_model_loader
|
||||
from tensorpack.predict.config import PredictConfig
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
from itertools import groupby
|
||||
from threading import Lock
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from config import config as cfg
|
||||
from config import finalize_configs
|
||||
from dataset import register_coco
|
||||
from eval import DetectionResult, predict_image
|
||||
from modeling.generalized_rcnn import ResNetFPNModel
|
||||
from tensorpack.predict.base import OfflinePredictor
|
||||
from tensorpack.predict.config import PredictConfig
|
||||
from tensorpack.tfutils.sessinit import get_model_loader
|
||||
|
||||
|
||||
class MaskRCNNService:
|
||||
|
||||
@@ -26,7 +27,7 @@ class MaskRCNNService:
|
||||
# class method to load trained model and create an offline predictor
|
||||
@classmethod
|
||||
def get_predictor(cls):
|
||||
""" load trained model"""
|
||||
"""load trained model"""
|
||||
|
||||
with cls.lock:
|
||||
# check if model is already loaded
|
||||
@@ -139,11 +140,10 @@ class MaskRCNNService:
|
||||
# create predictor
|
||||
MaskRCNNService.get_predictor()
|
||||
|
||||
import json
|
||||
from flask import Flask
|
||||
from flask import request
|
||||
from flask import Response
|
||||
import base64
|
||||
import json
|
||||
|
||||
from flask import Flask, Response, request
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
+1
@@ -12,6 +12,7 @@
|
||||
# timeout MODEL_SERVER_TIMEOUT 70 seconds
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
|
||||
+3
-4
@@ -1,13 +1,12 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import glob
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
|
||||
@@ -15,15 +15,15 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import glob
|
||||
|
||||
|
||||
def copy_files(src, dest):
|
||||
|
||||
@@ -6,17 +6,15 @@
|
||||
# the root directory of this source tree. An additional grant of patent rights
|
||||
# can be found in the PATENTS file in the same directory.
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import json
|
||||
|
||||
from train_driver import main as single_process_main
|
||||
from fairseq import distributed_utils, options
|
||||
|
||||
from multiprocessing_train import ErrorHandler
|
||||
|
||||
import torch
|
||||
from fairseq import distributed_utils, options
|
||||
from multiprocessing_train import ErrorHandler
|
||||
from train_driver import main as single_process_main
|
||||
|
||||
|
||||
def run(args, error_queue):
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
from sagemaker_translate import model_fn, input_fn, output_fn, predict_fn
|
||||
|
||||
import flask
|
||||
import os
|
||||
|
||||
import flask
|
||||
from sagemaker_translate import input_fn, model_fn, output_fn, predict_fn
|
||||
|
||||
prefix = "/opt/ml/"
|
||||
model_path = os.path.join(prefix, "model")
|
||||
|
||||
@@ -9,20 +9,18 @@
|
||||
Translate raw text with a trained model. Batches data on-the-fly.
|
||||
"""
|
||||
|
||||
from collections import namedtuple
|
||||
import numpy as np
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections import namedtuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from fairseq import data, options, tasks, tokenizer, utils
|
||||
from fairseq.sequence_generator import SequenceGenerator
|
||||
|
||||
|
||||
Batch = namedtuple("Batch", "srcs tokens lengths")
|
||||
Translation = namedtuple("Translation", "src_str hypos pos_scores alignments")
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
# timeout MODEL_SERVER_TIMEOUT 60 seconds
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
|
||||
@@ -1,22 +1,18 @@
|
||||
#!/opt/conda/bin/python
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
|
||||
import os
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import shutil
|
||||
import sys
|
||||
import traceback
|
||||
import copy
|
||||
import shutil
|
||||
from glob import glob
|
||||
|
||||
from fairseq import distributed_utils, options
|
||||
|
||||
from train_driver import main
|
||||
|
||||
|
||||
# These are the paths to where SageMaker mounts interesting things in your container.
|
||||
|
||||
prefix = '/opt/ml/'
|
||||
|
||||
@@ -14,14 +14,14 @@ Train a new model on one or across multiple GPUs.
|
||||
|
||||
import collections
|
||||
import itertools
|
||||
import os
|
||||
import math
|
||||
import torch
|
||||
import os
|
||||
|
||||
import torch
|
||||
from fairseq import distributed_utils, options, progress_bar, tasks, utils
|
||||
from fairseq.data import iterators
|
||||
from fairseq.trainer import Trainer
|
||||
from fairseq.meters import AverageMeter, StopwatchMeter
|
||||
from fairseq.trainer import Trainer
|
||||
|
||||
|
||||
def main(args):
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
import argparse
|
||||
import logging
|
||||
import sagemaker_containers
|
||||
import requests
|
||||
|
||||
import os
|
||||
import io
|
||||
import glob
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import requests
|
||||
import sagemaker_containers
|
||||
from fastai.vision import *
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
+10
-14
@@ -1,23 +1,19 @@
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import time
|
||||
import sys
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import csv
|
||||
import sys
|
||||
import time
|
||||
|
||||
import boto3
|
||||
|
||||
from awsglue.utils import getResolvedOptions
|
||||
|
||||
import pyspark
|
||||
from pyspark.sql import SparkSession
|
||||
from pyspark.sql.types import IntegerType, StructField, StructType, StringType
|
||||
from pyspark.ml.feature import Tokenizer
|
||||
from pyspark.sql.functions import *
|
||||
from mleap.pyspark.spark_support import SimpleSparkSerializer
|
||||
|
||||
from awsglue.utils import getResolvedOptions
|
||||
from mleap.pyspark.spark_support import SimpleSparkSerializer
|
||||
from pyspark.ml.feature import Tokenizer
|
||||
from pyspark.sql import SparkSession
|
||||
from pyspark.sql.functions import *
|
||||
from pyspark.sql.types import IntegerType, StringType, StructField, StructType
|
||||
|
||||
|
||||
def csv_line(data):
|
||||
|
||||
+11
-13
@@ -1,22 +1,20 @@
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import time
|
||||
import sys
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import csv
|
||||
import sys
|
||||
import time
|
||||
|
||||
import boto3
|
||||
|
||||
from awsglue.utils import getResolvedOptions
|
||||
|
||||
import pyspark
|
||||
from pyspark.sql import SparkSession
|
||||
from pyspark.ml import Pipeline
|
||||
from pyspark.sql.types import StructField, StructType, StringType, DoubleType
|
||||
from pyspark.ml.feature import StringIndexer, VectorIndexer, OneHotEncoder, VectorAssembler
|
||||
from pyspark.sql.functions import *
|
||||
from awsglue.utils import getResolvedOptions
|
||||
from mleap.pyspark.spark_support import SimpleSparkSerializer
|
||||
from pyspark.ml import Pipeline
|
||||
from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler, VectorIndexer
|
||||
from pyspark.sql import SparkSession
|
||||
from pyspark.sql.functions import *
|
||||
from pyspark.sql.types import DoubleType, StringType, StructField, StructType
|
||||
|
||||
|
||||
def csv_line(data):
|
||||
|
||||
+16
-17
@@ -1,27 +1,26 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
import sys
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import csv
|
||||
import sys
|
||||
import time
|
||||
|
||||
import boto3
|
||||
|
||||
from awsglue.utils import getResolvedOptions
|
||||
|
||||
import pyspark
|
||||
from pyspark.sql import SparkSession
|
||||
from pyspark.ml import Pipeline
|
||||
from pyspark.ml.feature import (
|
||||
StringIndexer,
|
||||
VectorIndexer,
|
||||
OneHotEncoder,
|
||||
VectorAssembler,
|
||||
IndexToString,
|
||||
)
|
||||
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
|
||||
from pyspark.sql.functions import *
|
||||
from awsglue.utils import getResolvedOptions
|
||||
from mleap.pyspark.spark_support import SimpleSparkSerializer
|
||||
from pyspark.ml import Pipeline
|
||||
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
|
||||
from pyspark.ml.feature import (
|
||||
IndexToString,
|
||||
OneHotEncoder,
|
||||
StringIndexer,
|
||||
VectorAssembler,
|
||||
VectorIndexer,
|
||||
)
|
||||
from pyspark.sql import SparkSession
|
||||
from pyspark.sql.functions import *
|
||||
|
||||
|
||||
def toCSVLine(data):
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
Custom Framework Estimator for JAX
|
||||
"""
|
||||
from sagemaker.estimator import Framework
|
||||
from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
|
||||
from sagemaker.tensorflow.model import TensorFlowModel
|
||||
from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT
|
||||
|
||||
|
||||
class JaxEstimator(Framework):
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
"""
|
||||
Train JAX model and serialize as TF SavedModel
|
||||
"""
|
||||
import time
|
||||
import argparse
|
||||
import functools
|
||||
import time
|
||||
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
|
||||
@@ -15,8 +15,8 @@ Train Trax model and serialize as TF SavedModel
|
||||
"""
|
||||
import argparse
|
||||
|
||||
import trax
|
||||
import tensorflow as tf
|
||||
import trax
|
||||
from trax import layers as tl
|
||||
from trax.supervised import training
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import shlex
|
||||
import os
|
||||
from retrying import retry
|
||||
from subprocess import CalledProcessError
|
||||
|
||||
from retrying import retry
|
||||
from sagemaker_inference import model_server
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
"""
|
||||
ModelHandler defines an example model handler for load and inference requests for MXNet CPU models
|
||||
"""
|
||||
from collections import namedtuple
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import namedtuple
|
||||
|
||||
import mxnet as mx
|
||||
import numpy as np
|
||||
|
||||
+11
-13
@@ -1,23 +1,16 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
import sys
|
||||
from io import StringIO
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from io import StringIO
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from sklearn.compose import ColumnTransformer
|
||||
from sklearn.externals import joblib
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import Binarizer, StandardScaler, OneHotEncoder
|
||||
|
||||
from sagemaker_containers.beta.framework import (
|
||||
content_types,
|
||||
encoders,
|
||||
@@ -26,6 +19,11 @@ from sagemaker_containers.beta.framework import (
|
||||
transformer,
|
||||
worker,
|
||||
)
|
||||
from sklearn.compose import ColumnTransformer
|
||||
from sklearn.externals import joblib
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import Binarizer, OneHotEncoder, StandardScaler
|
||||
|
||||
# Since we get a headerless CSV file we specify the column names here.
|
||||
feature_columns_names = [
|
||||
|
||||
@@ -11,7 +11,6 @@ import torch
|
||||
# Local Dependencies:
|
||||
from model import MNISTNet
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.addHandler(logging.StreamHandler(sys.stdout))
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# Python Built-Ins:
|
||||
import argparse
|
||||
from distutils.dir_util import copy_tree
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
@@ -8,22 +7,22 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from distutils.dir_util import copy_tree
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
# External Dependencies:
|
||||
import numpy as np
|
||||
from packaging import version as pkgversion
|
||||
from sagemaker_pytorch_serving_container import handler_service as default_handler_service
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.optim as optim
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
from model import MNISTNet
|
||||
from packaging import version as pkgversion
|
||||
from sagemaker_pytorch_serving_container import handler_service as default_handler_service
|
||||
from torch.utils.data import DataLoader, Dataset
|
||||
|
||||
# Local Dependencies:
|
||||
from inference import *
|
||||
from model import MNISTNet
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import logging
|
||||
|
||||
import gzip
|
||||
import mxnet as mx
|
||||
import numpy as np
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
|
||||
import mxnet as mx
|
||||
import numpy as np
|
||||
|
||||
|
||||
def find_file(root_path, file_name):
|
||||
for root, dirs, files in os.walk(root_path):
|
||||
|
||||
+2
-3
@@ -10,15 +10,15 @@
|
||||
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
||||
# ANY KIND, either express or implied. See the License for the specific
|
||||
# language governing permissions and limitations under the License.
|
||||
import ast
|
||||
import argparse
|
||||
import ast
|
||||
import logging
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.nn.parallel
|
||||
import torch.optim
|
||||
import torch.utils.data
|
||||
@@ -26,7 +26,6 @@ import torch.utils.data.distributed
|
||||
import torchvision
|
||||
import torchvision.models
|
||||
import torchvision.transforms as transforms
|
||||
import torch.nn.functional as F
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
@@ -10,11 +10,11 @@
|
||||
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
||||
# ANY KIND, either express or implied. See the License for the specific
|
||||
# language governing permissions and limitations under the License.
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchvision
|
||||
import torchvision.transforms as transforms
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
classes = ("plane", "car", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck")
|
||||
|
||||
|
||||
@@ -3,16 +3,15 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import pickle
|
||||
import io
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import signal
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import flask
|
||||
|
||||
import pandas as pd
|
||||
|
||||
prefix = "/opt/ml/"
|
||||
|
||||
@@ -6,14 +6,13 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from sklearn import tree
|
||||
|
||||
# These are the paths to where SageMaker mounts interesting things in your container.
|
||||
|
||||
@@ -10,17 +10,14 @@
|
||||
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
||||
# ANY KIND, either express or implied. See the License for the specific
|
||||
# language governing permissions and limitations under the License.
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import os
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
import resnet_model
|
||||
import tensorflow as tf
|
||||
|
||||
INPUT_TENSOR_NAME = "inputs"
|
||||
SIGNATURE_NAME = "serving_default"
|
||||
|
||||
@@ -29,9 +29,7 @@ The key difference of the full preactivation 'v2' variant compared to the
|
||||
'v1' variant in [1] is the use of batch normalization before every weight layer
|
||||
rather than after.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
@@ -19,10 +19,10 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
# These are the paths to where SageMaker mounts interesting things in your container.
|
||||
|
||||
+6
-8
@@ -16,9 +16,7 @@ Generates tf.train.Example protos and writes them to TFRecord files from the
|
||||
python version of the CIFAR-10 dataset downloaded from
|
||||
https://www.cs.toronto.edu/~kriz/cifar.html.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
@@ -26,12 +24,12 @@ import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
from six.moves import cPickle as pickle
|
||||
from six.moves import urllib
|
||||
from six.moves import xrange # pylint: disable=redefined-builtin
|
||||
from ipywidgets import FloatProgress
|
||||
from IPython.display import display
|
||||
import tensorflow as tf
|
||||
from IPython.display import display
|
||||
from ipywidgets import FloatProgress
|
||||
from six.moves import cPickle as pickle
|
||||
from six.moves import xrange # pylint: disable=redefined-builtin
|
||||
from six.moves import urllib
|
||||
|
||||
CIFAR_FILENAME = "cifar-10-python.tar.gz"
|
||||
CIFAR_DOWNLOAD_URL = "https://www.cs.toronto.edu/~kriz/" + CIFAR_FILENAME
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import argparse
|
||||
import numpy as np
|
||||
import os
|
||||
import tensorflow as tf
|
||||
from mlagents_envs.environment import UnityEnvironment
|
||||
import mlagents
|
||||
import subprocess
|
||||
import yaml
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import mlagents
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
import yaml
|
||||
from mlagents_envs.environment import UnityEnvironment
|
||||
|
||||
|
||||
def parse_args():
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import time
|
||||
import sys
|
||||
from io import StringIO
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from io import StringIO
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from sklearn.externals import joblib
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.svm import SVR
|
||||
from sklearn.feature_selection import f_regression, mutual_info_regression, SelectKBest, RFE
|
||||
|
||||
from sagemaker_containers.beta.framework import (
|
||||
content_types,
|
||||
encoders,
|
||||
@@ -26,6 +19,11 @@ from sagemaker_containers.beta.framework import (
|
||||
transformer,
|
||||
worker,
|
||||
)
|
||||
from sklearn.externals import joblib
|
||||
from sklearn.feature_selection import RFE, SelectKBest, f_regression, mutual_info_regression
|
||||
from sklearn.impute import SimpleImputer
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.svm import SVR
|
||||
|
||||
label_column = "y"
|
||||
INPUT_FEATURES_SIZE = 100
|
||||
|
||||
@@ -3,17 +3,16 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import StringIO
|
||||
import sys
|
||||
import signal
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import flask
|
||||
|
||||
import pandas as pd
|
||||
import StringIO
|
||||
|
||||
prefix = "/opt/ml/"
|
||||
model_path = os.path.join(prefix, "model")
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
# timeout MODEL_SERVER_TIMEOUT 60 seconds
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
|
||||
@@ -6,17 +6,16 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from numpy import mean
|
||||
from sklearn import tree
|
||||
from sklearn.model_selection import cross_val_score
|
||||
from numpy import mean
|
||||
|
||||
# These are the paths to where SageMaker mounts interesting things in your container.
|
||||
|
||||
|
||||
+3
-2
@@ -10,9 +10,10 @@
|
||||
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
||||
# express or implied. See the License for the specific language governing
|
||||
# permissions and limitations under the License.
|
||||
import boto3
|
||||
from datetime import datetime, timedelta
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import boto3
|
||||
|
||||
client = boto3.client("sagemaker")
|
||||
running_jobs = client.list_training_jobs(CreationTimeAfter=datetime.utcnow() - timedelta(hours=1))
|
||||
|
||||
@@ -15,24 +15,24 @@ from __future__ import absolute_import, division, print_function
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import os
|
||||
import re
|
||||
|
||||
import keras
|
||||
import tensorflow as tf
|
||||
from keras import backend as K
|
||||
from keras.callbacks import TensorBoard, ModelCheckpoint
|
||||
from keras.callbacks import ModelCheckpoint, TensorBoard
|
||||
from keras.layers import (
|
||||
Activation,
|
||||
BatchNormalization,
|
||||
Conv2D,
|
||||
Dense,
|
||||
Dropout,
|
||||
Flatten,
|
||||
MaxPooling2D,
|
||||
BatchNormalization,
|
||||
)
|
||||
from keras.models import Sequential
|
||||
from keras.optimizers import Adam, SGD, RMSprop
|
||||
from keras.optimizers import SGD, Adam, RMSprop
|
||||
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
tf.logging.set_verbosity(tf.logging.INFO)
|
||||
|
||||
@@ -12,8 +12,8 @@ from itertools import chain, islice
|
||||
|
||||
import mxnet as mx
|
||||
import numpy as np
|
||||
from mxnet import gluon, autograd, nd
|
||||
from mxnet.io import DataIter, DataBatch, DataDesc
|
||||
from mxnet import autograd, gluon, nd
|
||||
from mxnet.io import DataBatch, DataDesc, DataIter
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import print_function
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
import mxnet as mx
|
||||
import mxnet.contrib.onnx as onnx_mxnet
|
||||
import numpy as np
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
import random
|
||||
|
||||
|
||||
# sample preprocess_handler (to be implemented by customer)
|
||||
# This is a trivial example, where we demonstrate an echo preprocessor for json data
|
||||
# for others though, we are generating random data (real customers would not do that obviously/hopefully)
|
||||
|
||||
@@ -16,14 +16,13 @@ import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pandas as pd
|
||||
import pickle as pkl
|
||||
|
||||
from sagemaker_containers import entry_point
|
||||
from sagemaker_xgboost_container.data_utils import get_dmatrix
|
||||
from sagemaker_xgboost_container import distributed
|
||||
|
||||
import pandas as pd
|
||||
import xgboost as xgb
|
||||
from sagemaker_containers import entry_point
|
||||
from sagemaker_xgboost_container import distributed
|
||||
from sagemaker_xgboost_container.data_utils import get_dmatrix
|
||||
|
||||
|
||||
def _xgb_train(params, dtrain, evals, num_boost_round, model_dir, is_master):
|
||||
|
||||
@@ -15,12 +15,11 @@ from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pandas as pd
|
||||
|
||||
import pandas as pd
|
||||
from sklearn import tree
|
||||
from sklearn.externals import joblib
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Standard Library
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
|
||||
# Third Party
|
||||
import mxnet as mx
|
||||
@@ -7,8 +9,6 @@ import numpy as np
|
||||
from mxnet import autograd, gluon, init
|
||||
from mxnet.gluon import nn
|
||||
from mxnet.gluon.data.vision import datasets, transforms
|
||||
import os
|
||||
import time
|
||||
|
||||
|
||||
def parse_args():
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import numpy as np
|
||||
import math
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from matplotlib.animation import FuncAnimation
|
||||
|
||||
plt.rcParams.update({"font.size": 8})
|
||||
|
||||
@@ -11,14 +11,13 @@ For more information, please refer to https://github.com/awslabs/sagemaker-debug
|
||||
|
||||
# Standard Library
|
||||
import argparse
|
||||
import logging
|
||||
import random
|
||||
|
||||
# Third Party
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
import smdebug.tensorflow as smd
|
||||
|
||||
import logging
|
||||
import tensorflow as tf
|
||||
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
|
||||
+1
-2
@@ -8,14 +8,13 @@ For more information, please refer to https://github.com/awslabs/sagemaker-debug
|
||||
|
||||
# Standard Library
|
||||
import argparse
|
||||
import logging
|
||||
import random
|
||||
|
||||
# Third Party
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
|
||||
import logging
|
||||
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
+7
-4
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
|
||||
import tensorflow as tf
|
||||
from tensorflow.python.estimator.model_fn import ModeKeys as Modes
|
||||
|
||||
@@ -110,9 +111,10 @@ def _input_fn(training_dir, training_filename, batch_size=100):
|
||||
|
||||
|
||||
def neo_preprocess(payload, content_type):
|
||||
import logging
|
||||
import numpy as np
|
||||
import io
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
logging.info("Invoking user-defined pre-processing function")
|
||||
|
||||
@@ -132,9 +134,10 @@ def neo_preprocess(payload, content_type):
|
||||
|
||||
### NOTE: this function cannot use MXNet
|
||||
def neo_postprocess(result):
|
||||
import logging
|
||||
import numpy as np
|
||||
import json
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
logging.info("Invoking user-defined post-processing function")
|
||||
|
||||
|
||||
+1
@@ -1,5 +1,6 @@
|
||||
"""Converts MNIST data to TFRecords file format with Example protos."""
|
||||
import os
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
|
||||
+7
-7
@@ -1,9 +1,10 @@
|
||||
import io
|
||||
import os
|
||||
import PIL.Image
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
@@ -12,14 +13,13 @@ logger.setLevel(logging.DEBUG)
|
||||
# Training methods #
|
||||
# ------------------------------------------------------------ #
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import time
|
||||
import argparse
|
||||
import warnings
|
||||
|
||||
import mxnet as mx
|
||||
from mxnet import nd
|
||||
from mxnet import gluon
|
||||
from mxnet import autograd
|
||||
from mxnet import autograd, gluon, nd
|
||||
|
||||
|
||||
def parse_args():
|
||||
@@ -74,7 +74,7 @@ def get_dataloader(net, data_shape, batch_size, num_workers, ctx):
|
||||
"""Get dataloader."""
|
||||
|
||||
from gluoncv import data as gdata
|
||||
from gluoncv.data.batchify import Tuple, Stack, Pad
|
||||
from gluoncv.data.batchify import Pad, Stack, Tuple
|
||||
from gluoncv.data.transforms.presets.ssd import SSDDefaultTrainTransform
|
||||
|
||||
width, height = data_shape, data_shape
|
||||
|
||||
+2
-1
@@ -15,9 +15,10 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from imdb import Imdb
|
||||
import random
|
||||
|
||||
from imdb import Imdb
|
||||
|
||||
|
||||
class ConcatDB(Imdb):
|
||||
"""
|
||||
|
||||
+5
-3
@@ -18,18 +18,20 @@
|
||||
# under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
curr_path = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.append(os.path.join(curr_path, "../python"))
|
||||
import mxnet as mx
|
||||
import random
|
||||
import argparse
|
||||
import cv2
|
||||
import random
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import cv2
|
||||
import mxnet as mx
|
||||
|
||||
try:
|
||||
import multiprocessing
|
||||
except ImportError:
|
||||
|
||||
+2
-1
@@ -15,9 +15,10 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import numpy as np
|
||||
import os.path as osp
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
class Imdb(object):
|
||||
"""
|
||||
|
||||
+4
-2
@@ -16,11 +16,13 @@
|
||||
# under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from imdb import Imdb
|
||||
import xml.etree.ElementTree as ET
|
||||
import cv2
|
||||
|
||||
|
||||
class PascalVoc(Imdb):
|
||||
|
||||
+4
-2
@@ -19,14 +19,16 @@
|
||||
# under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import sys, os
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
curr_path = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.append(os.path.join(curr_path, ".."))
|
||||
from pascal_voc import PascalVoc
|
||||
from concat_db import ConcatDB
|
||||
from pascal_voc import PascalVoc
|
||||
|
||||
|
||||
def load_pascal(image_set, year, devkit_path, shuffle=False):
|
||||
|
||||
+5
-4
@@ -1,12 +1,13 @@
|
||||
import numpy as np
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
import mxnet as mx
|
||||
|
||||
# Please make sure to import neomx
|
||||
import neomx # noqa: F401
|
||||
import io
|
||||
import os
|
||||
import logging
|
||||
import numpy as np
|
||||
|
||||
# Change the context to mx.gpu() if deploying to a GPU endpoint
|
||||
ctx = mx.cpu()
|
||||
|
||||
+1
-1
@@ -4,9 +4,9 @@ import logging
|
||||
import os
|
||||
import pickle
|
||||
|
||||
import neopytorch
|
||||
import numpy as np
|
||||
import torch
|
||||
import neopytorch
|
||||
import torchvision.transforms as transforms
|
||||
from PIL import Image # Training container doesn't have this package
|
||||
|
||||
|
||||
+1
-1
@@ -4,9 +4,9 @@ import logging
|
||||
import os
|
||||
import pickle
|
||||
|
||||
import neopytorch
|
||||
import numpy as np
|
||||
import torch
|
||||
import neopytorch
|
||||
import torchvision.transforms as transforms
|
||||
from PIL import Image # Training container doesn't have this package
|
||||
|
||||
|
||||
+1
-1
@@ -4,9 +4,9 @@ import logging
|
||||
import os
|
||||
import pickle
|
||||
|
||||
import neopytorch
|
||||
import numpy as np
|
||||
import torch
|
||||
import neopytorch
|
||||
import torchvision.transforms as transforms
|
||||
from PIL import Image # Training container doesn't have this package
|
||||
|
||||
|
||||
+7
-4
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
|
||||
import tensorflow as tf
|
||||
from tensorflow.python.estimator.model_fn import ModeKeys as Modes
|
||||
|
||||
@@ -111,9 +112,10 @@ def _input_fn(training_dir, training_filename, batch_size=100):
|
||||
|
||||
|
||||
def neo_preprocess(payload, content_type):
|
||||
import logging
|
||||
import numpy as np
|
||||
import io
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
logging.info("Invoking user-defined pre-processing function")
|
||||
|
||||
@@ -133,9 +135,10 @@ def neo_preprocess(payload, content_type):
|
||||
|
||||
### NOTE: this function cannot use MXNet
|
||||
def neo_postprocess(result):
|
||||
import logging
|
||||
import numpy as np
|
||||
import json
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
|
||||
logging.info("Invoking user-defined post-processing function")
|
||||
|
||||
|
||||
+1
@@ -1,5 +1,6 @@
|
||||
"""Converts MNIST data to TFRecords file format with Example protos."""
|
||||
import os
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
|
||||
+2
-5
@@ -10,17 +10,14 @@
|
||||
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
||||
# ANY KIND, either express or implied. See the License for the specific
|
||||
# language governing permissions and limitations under the License.
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import os
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
import resnet_model
|
||||
import tensorflow as tf
|
||||
|
||||
INPUT_TENSOR_NAME = "inputs"
|
||||
SIGNATURE_NAME = "serving_default"
|
||||
|
||||
+1
-3
@@ -29,9 +29,7 @@ The key difference of the full preactivation 'v2' variant compared to the
|
||||
'v1' variant in [1] is the use of batch normalization before every weight layer
|
||||
rather than after.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
+2
-2
@@ -19,10 +19,10 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
# These are the paths to where SageMaker mounts interesting things in your container.
|
||||
|
||||
+6
-8
@@ -16,9 +16,7 @@ Generates tf.train.Example protos and writes them to TFRecord files from the
|
||||
python version of the CIFAR-10 dataset downloaded from
|
||||
https://www.cs.toronto.edu/~kriz/cifar.html.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
@@ -26,12 +24,12 @@ import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
from six.moves import cPickle as pickle
|
||||
from six.moves import urllib
|
||||
from six.moves import xrange # pylint: disable=redefined-builtin
|
||||
from ipywidgets import FloatProgress
|
||||
from IPython.display import display
|
||||
import tensorflow as tf
|
||||
from IPython.display import display
|
||||
from ipywidgets import FloatProgress
|
||||
from six.moves import cPickle as pickle
|
||||
from six.moves import xrange # pylint: disable=redefined-builtin
|
||||
from six.moves import urllib
|
||||
|
||||
CIFAR_FILENAME = "cifar-10-python.tar.gz"
|
||||
CIFAR_DOWNLOAD_URL = "https://www.cs.toronto.edu/~kriz/" + CIFAR_FILENAME
|
||||
|
||||
+3
-4
@@ -3,16 +3,15 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
from io import StringIO
|
||||
import sys
|
||||
import signal
|
||||
import sys
|
||||
import traceback
|
||||
from io import StringIO
|
||||
|
||||
import flask
|
||||
|
||||
import pandas as pd
|
||||
import xgboost
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
# timeout MODEL_SERVER_TIMEOUT 60 seconds
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
|
||||
+2
-2
@@ -6,15 +6,15 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import pandas as pd
|
||||
import xgboost as xgb
|
||||
import smdebug.xgboost as smd
|
||||
import xgboost as xgb
|
||||
|
||||
print("Libraries imported")
|
||||
|
||||
|
||||
+27
-34
@@ -1,44 +1,26 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
from string import Template
|
||||
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
import multiprocessing
|
||||
import signal
|
||||
import subprocess
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from utils import ExitSignalHandler
|
||||
from utils import (
|
||||
write_failure_file,
|
||||
print_json_object,
|
||||
load_json_object,
|
||||
save_model_artifacts,
|
||||
print_files_in_path,
|
||||
)
|
||||
|
||||
import traceback
|
||||
from io import StringIO
|
||||
import os
|
||||
import shutil
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import decimal
|
||||
import json
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from joblib import dump, load
|
||||
from sklearn.feature_extraction.text import CountVectorizer
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
import multiprocessing
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from io import StringIO
|
||||
from string import Template
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import boto3
|
||||
import decimal
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from joblib import dump, load
|
||||
from sagemaker_containers.beta.framework import (
|
||||
content_types,
|
||||
encoders,
|
||||
@@ -47,6 +29,17 @@ from sagemaker_containers.beta.framework import (
|
||||
transformer,
|
||||
worker,
|
||||
)
|
||||
from sklearn.feature_extraction.text import CountVectorizer
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
|
||||
from utils import (
|
||||
ExitSignalHandler,
|
||||
load_json_object,
|
||||
print_files_in_path,
|
||||
print_json_object,
|
||||
save_model_artifacts,
|
||||
write_failure_file,
|
||||
)
|
||||
|
||||
cpu_count = multiprocessing.cpu_count()
|
||||
|
||||
|
||||
+13
-12
@@ -3,21 +3,22 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import stat
|
||||
import json
|
||||
import shutil
|
||||
import flask
|
||||
from flask import Flask, jsonify, request, Response
|
||||
import glob
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import random
|
||||
import csv
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
from io import StringIO
|
||||
from joblib import dump, load
|
||||
|
||||
import boto3
|
||||
import flask
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from flask import Flask, Response, jsonify, request
|
||||
from joblib import dump, load
|
||||
from sagemaker_containers.beta.framework import (
|
||||
content_types,
|
||||
encoders,
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import signal
|
||||
import pprint
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
import signal
|
||||
from os import path
|
||||
|
||||
|
||||
|
||||
+26
-34
@@ -1,44 +1,25 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
from string import Template
|
||||
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
import shutil
|
||||
import multiprocessing
|
||||
import signal
|
||||
import subprocess
|
||||
|
||||
from utils import ExitSignalHandler
|
||||
from utils import (
|
||||
write_failure_file,
|
||||
print_json_object,
|
||||
load_json_object,
|
||||
save_model_artifacts,
|
||||
print_files_in_path,
|
||||
)
|
||||
|
||||
|
||||
import traceback
|
||||
from io import StringIO
|
||||
import os
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import decimal
|
||||
import json
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from joblib import dump, load
|
||||
from sklearn.feature_extraction.text import CountVectorizer
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
import multiprocessing
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from io import StringIO
|
||||
from string import Template
|
||||
|
||||
import boto3
|
||||
import decimal
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from joblib import dump, load
|
||||
from sagemaker_containers.beta.framework import (
|
||||
content_types,
|
||||
encoders,
|
||||
@@ -47,6 +28,17 @@ from sagemaker_containers.beta.framework import (
|
||||
transformer,
|
||||
worker,
|
||||
)
|
||||
from sklearn.feature_extraction.text import CountVectorizer
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
|
||||
from utils import (
|
||||
ExitSignalHandler,
|
||||
load_json_object,
|
||||
print_files_in_path,
|
||||
print_json_object,
|
||||
save_model_artifacts,
|
||||
write_failure_file,
|
||||
)
|
||||
|
||||
hyperparameters_file_path = "/opt/ml/input/config/hyperparameters.json"
|
||||
inputdataconfig_file_path = "/opt/ml/input/config/inputdataconfig.json"
|
||||
|
||||
+15
-13
@@ -3,18 +3,19 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import stat
|
||||
import json
|
||||
import shutil
|
||||
import flask
|
||||
from flask import Flask, jsonify, request, make_response, Response
|
||||
import glob
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import csv
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
from io import StringIO
|
||||
|
||||
import flask
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from flask import Flask, Response, jsonify, make_response, request
|
||||
from joblib import dump, load
|
||||
from sagemaker_containers.beta.framework import (
|
||||
content_types,
|
||||
@@ -24,12 +25,13 @@ from sagemaker_containers.beta.framework import (
|
||||
transformer,
|
||||
worker,
|
||||
)
|
||||
|
||||
from utils import (
|
||||
write_failure_file,
|
||||
print_json_object,
|
||||
load_json_object,
|
||||
save_model_artifacts,
|
||||
print_files_in_path,
|
||||
print_json_object,
|
||||
save_model_artifacts,
|
||||
write_failure_file,
|
||||
)
|
||||
|
||||
model_artifacts_path = "/opt/ml/model/"
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
import signal
|
||||
import pprint
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
import signal
|
||||
from os import path
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import argparse
|
||||
import random
|
||||
import csv
|
||||
import random
|
||||
|
||||
parser = argparse.ArgumentParser(description="Generate sample data")
|
||||
parser.add_argument("--samples", type=int, default=10000, help="Number of samples to generate")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import argparse
|
||||
import csv
|
||||
|
||||
import boto3
|
||||
|
||||
parser = argparse.ArgumentParser(description="Load DynamoDB data")
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import time
|
||||
import boto3
|
||||
import argparse
|
||||
import pandas as pd
|
||||
import pathlib
|
||||
import time
|
||||
|
||||
import boto3
|
||||
import pandas as pd
|
||||
|
||||
# Parse argument variables passed via the CreateDataset processing step
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import boto3
|
||||
import time
|
||||
|
||||
import boto3
|
||||
|
||||
|
||||
def delete_project_resources(
|
||||
sagemaker_boto_client,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import time
|
||||
import boto3
|
||||
import argparse
|
||||
import time
|
||||
|
||||
import boto3
|
||||
|
||||
# Parse argument variables passed via the DeployModel processing step
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import os
|
||||
import sys
|
||||
import pickle
|
||||
import xgboost as xgb
|
||||
import argparse
|
||||
import pandas as pd
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
import xgboost as xgb
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
@@ -15,15 +15,14 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import logging
|
||||
|
||||
from mxnet import gluon
|
||||
import mxnet as mx
|
||||
|
||||
import numpy as np
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
import mxnet as mx
|
||||
import numpy as np
|
||||
from mxnet import gluon
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
|
||||
|
||||
@@ -13,16 +13,18 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from inference import transform_fn, model_fn
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
|
||||
import boto3
|
||||
import botocore
|
||||
import tarfile
|
||||
import numpy as np
|
||||
import sagemaker
|
||||
|
||||
from inference import model_fn, transform_fn
|
||||
|
||||
|
||||
def fetch_model(model_data):
|
||||
"""Untar the model.tar.gz object either from local file system
|
||||
|
||||
@@ -13,12 +13,12 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from train import train, parse_args
|
||||
|
||||
import sys
|
||||
import os
|
||||
import boto3
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import boto3
|
||||
from train import parse_args, train
|
||||
|
||||
dirname = os.path.dirname(os.path.abspath(__file__))
|
||||
with open(os.path.join(dirname, "config.json"), "r") as f:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user