Autoencoder Anomaly Detection Using PyTorch 1.10 on Windows 11

Posted on May 17, 2022 by jamesdmccaffrey

Every few months I revisit my standard neural network examples to make sure that changes in the underlying code libraries (PyTorch, Keras/TensorFlow) haven’t introduced a breaking change(s). One of my standard examples is autoencoder anomaly detection.

The idea is to take a set of data and implement a deep neural network that predicts its input. The values of the interior hidden layer of nodes is a condensed representation of the input. The output nodes are a reconstruction of the input. Data items where the reconstructed input is very different from the associated input are anomalous in some way.

My demo uses a synthetic set of Employee data. There are five feature variables: employee sex (M, F), age, city (anaheim, boulder, concord), annual income, and job-type (mgmt, supp, tech). There are 240 items. The normalized and encoded data looks like:

# sex  age   city      income   job_type
 -1   0.27   0  1  0   0.7610   0  0  1
  1   0.19   0  0  1   0.6550   0  1  0
. . .

My demo network uses a 9-4-(2)-4-9 architecture. The input and output size is determined by the data, but the number of hidden layers and the number of nodes in each, are hyperparameters that must be determined by trial and error.


I love to observe people and things, especially in Las Vegas. On a recent trip to speak at a tech conference, I noticed that electronic versions of games such as Blackjack, Roulette, and Craps display results of recent games. This encourages players to seek out and bet on anomalies — results that appear less than expected or more than expected. Left: This craps game at the MGM Grand shows “hot” numbers and “cold” numbers. Right: The Fortune Cup horse race game shows the results of the most recent 40 races. Fascinating.

Demo code. Replace “lt”, “gt”, “lte”, “gte” with Boolean operator symbols — my weak blog editor chokes on symbols.

# employee_auto_anom.py
# autoencoder reconstruction error anomaly detection
# PyTorch 1.10.0-CPU Anaconda3-2020.02  Python 3.7.6
# Windows 10/11

import numpy as np
import torch as T

device = T.device('cpu') 

# -----------------------------------------------------------

class EmployeeDataset(T.utils.data.Dataset):
  # sex  age   city     income  job
  # -1   0.27  0  1  0  0.7610  0  0  1
  # +1   0.19  0  0  1  0.6550  0  1  0
  # sex: -1 = male, +1 = female
  # city: anaheim, boulder, concord
  # job: mgmt, supp, tech

  def __init__(self, src_file):
    tmp_x = np.loadtxt(src_file, usecols=range(0,9),
      delimiter="\t", comments="#", dtype=np.float32)
    self.x_data = T.tensor(tmp_x, dtype=T.float32).to(device)

  def __len__(self):
    return len(self.x_data)

  def __getitem__(self, idx):
    preds = self.x_data[idx, :]  # row idx, all cols
    sample = { 'predictors' : preds }  # as Dictionary
    return sample

# -----------------------------------------------------------

class Net(T.nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.enc1 = T.nn.Linear(9, 4)  # 9-4-2-4-9
    self.enc2 = T.nn.Linear(4, 2)

    self.dec1 = T.nn.Linear(2, 4)
    self.dec2 = T.nn.Linear(4, 9)

    T.nn.init.xavier_uniform_(self.enc1.weight)
    T.nn.init.zeros_(self.enc1.bias)
    T.nn.init.xavier_uniform_(self.enc2.weight)
    T.nn.init.zeros_(self.enc2.bias)
    T.nn.init.xavier_uniform_(self.dec1.weight)
    T.nn.init.zeros_(self.dec1.bias)
    T.nn.init.xavier_uniform_(self.dec2.weight)
    T.nn.init.zeros_(self.dec2.bias)

  def forward(self, x):
    z = T.tanh(self.enc1(x))
    z = T.tanh(self.enc2(z))
    z = T.tanh(self.dec1(z))
    z = self.dec2(z)  # no activation
    return z

# -----------------------------------------------------------

def analyze_error(model, ds):
  largest_err = 0.0
  worst_x = None
  worst_y = None
  n_features = len(ds[0]['predictors'])

  for i in range(len(ds)):
    X = ds[i]['predictors']
    with T.no_grad():
      Y = model(X)  # should be same as X
    err = T.sum((X-Y)*(X-Y)).item()  # SSE all features
    err = err / n_features           # sort of norm'ed SSE 

    if err "gt" largest_err:
      largest_err = err
      worst_x = X
      worst_y = Y

  np.set_printoptions(formatter={'float': '{: 0.4f}'.format})
  print("Largest reconstruction error: %0.4f" % largest_err)
  print("Worst data item    = ")
  print(worst_x.numpy())
  print("Its reconstruction = " )
  print(worst_y.numpy())

# -----------------------------------------------------------

def main():
  # 0. get started
  print("\nBegin Employee autoencoder anomaly detection ")
  T.manual_seed(2)
  np.random.seed(2)
  
  # 1. create DataLoader objects
  print("\nCreating Employee Dataset ")

  data_file = ".\\Data\\employee_all.txt"
  data_ds = EmployeeDataset(data_file)  # all 240 rows

  bat_size = 20
  data_ldr = T.utils.data.DataLoader(data_ds,
    batch_size=bat_size, shuffle=True)

  # 2. create network
  print("\nCreating 9-4-(2)-4-9 network ")
  net = Net().to(device)

# -----------------------------------------------------------

  # 3. train autoencoder model
  max_epochs = 1000
  ep_log_interval = 100
  lrn_rate = 0.005

  loss_func = T.nn.MSELoss()
  optimizer = T.optim.Adam(net.parameters(), lr=lrn_rate)

  print("\nbat_size = %3d " % bat_size)
  print("loss = " + str(loss_func))
  print("optimizer = Adam")
  print("lrn_rate = %0.3f " % lrn_rate)
  print("max_epochs = %3d " % max_epochs)
  

  print("\nStarting training")
  net.train()
  for epoch in range(0, max_epochs):
    epoch_loss = 0  # for one full epoch

    for (batch_idx, batch) in enumerate(data_ldr):
      X = batch['predictors'] 
      Y = batch['predictors'] 

      optimizer.zero_grad()
      oupt = net(X)
      loss_val = loss_func(oupt, Y)  # a tensor
      epoch_loss += loss_val.item()  # accumulate
      loss_val.backward()
      optimizer.step()

    if epoch % ep_log_interval == 0:
      print("epoch = %4d  |  loss = %0.4f" % \
       (epoch, epoch_loss))
  print("Done ")

# -----------------------------------------------------------

  # 4. find item with largest reconstruction error
  print("\nAnalyzing data for largest reconstruction error \n")
  net.eval()
  analyze_error(net, data_ds)

  print("\nEnd Employee autoencoder anomaly demo ")

if __name__ == "__main__":
  main()

Demo data:

# employee_all.txt
# sex (M = -1, F = +1), age / 100,
# city (anaheim = 100, boulder = 010, concord = 001),
# income / 100_00,
# job_type (mgmt = 100, supp = 010, tech = 001)
#
1	0.24	1	0	0	0.2950	0	0	1
-1	0.39	0	0	1	0.5120	0	1	0
1	0.63	0	1	0	0.7580	1	0	0
-1	0.36	1	0	0	0.4450	0	1	0
1	0.27	0	1	0	0.2860	0	0	1
1	0.50	0	1	0	0.5650	0	1	0
1	0.50	0	0	1	0.5500	0	1	0
-1	0.19	0	0	1	0.3270	1	0	0
1	0.22	0	1	0	0.2770	0	1	0
-1	0.39	0	0	1	0.4710	0	0	1
1	0.34	1	0	0	0.3940	0	1	0
-1	0.22	1	0	0	0.3350	1	0	0
1	0.35	0	0	1	0.3520	0	0	1
-1	0.33	0	1	0	0.4640	0	1	0
1	0.45	0	1	0	0.5410	0	1	0
1	0.42	0	1	0	0.5070	0	1	0
-1	0.33	0	1	0	0.4680	0	1	0
1	0.25	0	0	1	0.3000	0	1	0
-1	0.31	0	1	0	0.4640	1	0	0
1	0.27	1	0	0	0.3250	0	0	1
1	0.48	1	0	0	0.5400	0	1	0
-1	0.64	0	1	0	0.7130	0	0	1
1	0.61	0	1	0	0.7240	1	0	0
1	0.54	0	0	1	0.6100	1	0	0
1	0.29	1	0	0	0.3630	1	0	0
1	0.50	0	0	1	0.5500	0	1	0
1	0.55	0	0	1	0.6250	1	0	0
1	0.40	1	0	0	0.5240	1	0	0
1	0.22	1	0	0	0.2360	0	0	1
1	0.68	0	1	0	0.7840	1	0	0
-1	0.60	1	0	0	0.7170	0	0	1
-1	0.34	0	0	1	0.4650	0	1	0
-1	0.25	0	0	1	0.3710	1	0	0
-1	0.31	0	1	0	0.4890	0	1	0
1	0.43	0	0	1	0.4800	0	1	0
1	0.58	0	1	0	0.6540	0	0	1
-1	0.55	0	1	0	0.6070	0	0	1
-1	0.43	0	1	0	0.5110	0	1	0
-1	0.43	0	0	1	0.5320	0	1	0
-1	0.21	1	0	0	0.3720	1	0	0
1	0.55	0	0	1	0.6460	1	0	0
1	0.64	0	1	0	0.7480	1	0	0
-1	0.41	1	0	0	0.5880	0	1	0
1	0.64	0	0	1	0.7270	1	0	0
-1	0.56	0	0	1	0.6660	0	0	1
1	0.31	0	0	1	0.3600	0	1	0
-1	0.65	0	0	1	0.7010	0	0	1
1	0.55	0	0	1	0.6430	1	0	0
-1	0.25	1	0	0	0.4030	1	0	0
1	0.46	0	0	1	0.5100	0	1	0
-1	0.36	1	0	0	0.5350	1	0	0
1	0.52	0	1	0	0.5810	0	1	0
1	0.61	0	0	1	0.6790	1	0	0
1	0.57	0	0	1	0.6570	1	0	0
-1	0.46	0	1	0	0.5260	0	1	0
-1	0.62	1	0	0	0.6680	0	0	1
1	0.55	0	0	1	0.6270	1	0	0
-1	0.22	0	0	1	0.2770	0	1	0
-1	0.50	1	0	0	0.6290	1	0	0
-1	0.32	0	1	0	0.4180	0	1	0
-1	0.21	0	0	1	0.3560	1	0	0
1	0.44	0	1	0	0.5200	0	1	0
1	0.46	0	1	0	0.5170	0	1	0
1	0.62	0	1	0	0.6970	1	0	0
1	0.57	0	1	0	0.6640	1	0	0
-1	0.67	0	0	1	0.7580	0	0	1
1	0.29	1	0	0	0.3430	0	0	1
1	0.53	1	0	0	0.6010	1	0	0
-1	0.44	1	0	0	0.5480	0	1	0
1	0.46	0	1	0	0.5230	0	1	0
-1	0.20	0	1	0	0.3010	0	1	0
-1	0.38	1	0	0	0.5350	0	1	0
1	0.50	0	1	0	0.5860	0	1	0
1	0.33	0	1	0	0.4250	0	1	0
-1	0.33	0	1	0	0.3930	0	1	0
1	0.26	0	1	0	0.4040	1	0	0
1	0.58	1	0	0	0.7070	1	0	0
1	0.43	0	0	1	0.4800	0	1	0
-1	0.46	1	0	0	0.6440	1	0	0
1	0.60	1	0	0	0.7170	1	0	0
-1	0.42	1	0	0	0.4890	0	1	0
-1	0.56	0	0	1	0.5640	0	0	1
-1	0.62	0	1	0	0.6630	0	0	1
-1	0.50	1	0	0	0.6480	0	1	0
1	0.47	0	0	1	0.5200	0	1	0
-1	0.67	0	1	0	0.8040	0	0	1
-1	0.40	0	0	1	0.5040	0	1	0
1	0.42	0	1	0	0.4840	0	1	0
1	0.64	1	0	0	0.7200	1	0	0
-1	0.47	1	0	0	0.5870	0	0	1
1	0.45	0	1	0	0.5280	0	1	0
-1	0.25	0	0	1	0.4090	1	0	0
1	0.38	1	0	0	0.4840	1	0	0
1	0.55	0	0	1	0.6000	0	1	0
-1	0.44	1	0	0	0.6060	0	1	0
1	0.33	1	0	0	0.4100	0	1	0
1	0.34	0	0	1	0.3900	0	1	0
1	0.27	0	1	0	0.3370	0	0	1
1	0.32	0	1	0	0.4070	0	1	0
1	0.42	0	0	1	0.4700	0	1	0
-1	0.24	0	0	1	0.4030	1	0	0
1	0.42	0	1	0	0.5030	0	1	0
1	0.25	0	0	1	0.2800	0	0	1
1	0.51	0	1	0	0.5800	0	1	0
-1	0.55	0	1	0	0.6350	0	0	1
1	0.44	1	0	0	0.4780	0	0	1
-1	0.18	1	0	0	0.3980	1	0	0
-1	0.67	0	1	0	0.7160	0	0	1
1	0.45	0	0	1	0.5000	0	1	0
1	0.48	1	0	0	0.5580	0	1	0
-1	0.25	0	1	0	0.3900	0	1	0
-1	0.67	1	0	0	0.7830	0	1	0
1	0.37	0	0	1	0.4200	0	1	0
-1	0.32	1	0	0	0.4270	0	1	0
1	0.48	1	0	0	0.5700	0	1	0
-1	0.66	0	0	1	0.7500	0	0	1
1	0.61	1	0	0	0.7000	1	0	0
-1	0.58	0	0	1	0.6890	0	1	0
1	0.19	1	0	0	0.2400	0	0	1
1	0.38	0	0	1	0.4300	0	1	0
-1	0.27	1	0	0	0.3640	0	1	0
1	0.42	1	0	0	0.4800	0	1	0
1	0.60	1	0	0	0.7130	1	0	0
-1	0.27	0	0	1	0.3480	1	0	0
1	0.29	0	1	0	0.3710	1	0	0
-1	0.43	1	0	0	0.5670	0	1	0
1	0.48	1	0	0	0.5670	0	1	0
1	0.27	0	0	1	0.2940	0	0	1
-1	0.44	1	0	0	0.5520	1	0	0
1	0.23	0	1	0	0.2630	0	0	1
-1	0.36	0	1	0	0.5300	0	0	1
1	0.64	0	0	1	0.7250	1	0	0
1	0.29	0	0	1	0.3000	0	0	1
-1	0.33	1	0	0	0.4930	0	1	0
-1	0.66	0	1	0	0.7500	0	0	1
-1	0.21	0	0	1	0.3430	1	0	0
1	0.27	1	0	0	0.3270	0	0	1
1	0.29	1	0	0	0.3180	0	0	1
-1	0.31	1	0	0	0.4860	0	1	0
1	0.36	0	0	1	0.4100	0	1	0
1	0.49	0	1	0	0.5570	0	1	0
-1	0.28	1	0	0	0.3840	1	0	0
-1	0.43	0	0	1	0.5660	0	1	0
-1	0.46	0	1	0	0.5880	0	1	0
1	0.57	1	0	0	0.6980	1	0	0
-1	0.52	0	0	1	0.5940	0	1	0
-1	0.31	0	0	1	0.4350	0	1	0
-1	0.55	1	0	0	0.6200	0	0	1
1	0.50	1	0	0	0.5640	0	1	0
1	0.48	0	1	0	0.5590	0	1	0
-1	0.22	0	0	1	0.3450	1	0	0
1	0.59	0	0	1	0.6670	1	0	0
1	0.34	1	0	0	0.4280	0	0	1
-1	0.64	1	0	0	0.7720	0	0	1
1	0.29	0	0	1	0.3350	0	0	1
-1	0.34	0	1	0	0.4320	0	1	0
-1	0.61	1	0	0	0.7500	0	0	1
1	0.64	0	0	1	0.7110	1	0	0
-1	0.29	1	0	0	0.4130	1	0	0
1	0.63	0	1	0	0.7060	1	0	0
-1	0.29	0	1	0	0.4000	1	0	0
-1	0.51	1	0	0	0.6270	0	1	0
-1	0.24	0	0	1	0.3770	1	0	0
1	0.48	0	1	0	0.5750	0	1	0
1	0.18	1	0	0	0.2740	1	0	0
1	0.18	1	0	0	0.2030	0	0	1
1	0.33	0	1	0	0.3820	0	0	1
-1	0.20	0	0	1	0.3480	1	0	0
1	0.29	0	0	1	0.3300	0	0	1
-1	0.44	0	0	1	0.6300	1	0	0
-1	0.65	0	0	1	0.8180	1	0	0
-1	0.56	1	0	0	0.6370	0	0	1
-1	0.52	0	0	1	0.5840	0	1	0
-1	0.29	0	1	0	0.4860	1	0	0
-1	0.47	0	1	0	0.5890	0	1	0
1	0.68	1	0	0	0.7260	0	0	1
1	0.31	0	0	1	0.3600	0	1	0
1	0.61	0	1	0	0.6250	0	0	1
1	0.19	0	1	0	0.2150	0	0	1
1	0.38	0	0	1	0.4300	0	1	0
-1	0.26	1	0	0	0.4230	1	0	0
1	0.61	0	1	0	0.6740	1	0	0
1	0.40	1	0	0	0.4650	0	1	0
-1	0.49	1	0	0	0.6520	0	1	0
1	0.56	1	0	0	0.6750	1	0	0
-1	0.48	0	1	0	0.6600	0	1	0
1	0.52	1	0	0	0.5630	0	0	1
-1	0.18	1	0	0	0.2980	1	0	0
-1	0.56	0	0	1	0.5930	0	0	1
-1	0.52	0	1	0	0.6440	0	1	0
-1	0.18	0	1	0	0.2860	0	1	0
-1	0.58	1	0	0	0.6620	0	0	1
-1	0.39	0	1	0	0.5510	0	1	0
-1	0.46	1	0	0	0.6290	0	1	0
-1	0.40	0	1	0	0.4620	0	1	0
-1	0.60	1	0	0	0.7270	0	0	1
1	0.36	0	1	0	0.4070	0	0	1
1	0.44	1	0	0	0.5230	0	1	0
1	0.28	1	0	0	0.3130	0	0	1
1	0.54	0	0	1	0.6260	1	0	0
-1	0.51	1	0	0	0.6120	0	1	0
-1	0.32	0	1	0	0.4610	0	1	0
1	0.55	1	0	0	0.6270	1	0	0
1	0.25	0	0	1	0.2620	0	0	1
1	0.33	0	0	1	0.3730	0	0	1
-1	0.29	0	1	0	0.4620	1	0	0
1	0.65	1	0	0	0.7270	1	0	0
-1	0.43	0	1	0	0.5140	0	1	0
-1	0.54	0	1	0	0.6480	0	0	1
1	0.61	0	1	0	0.7270	1	0	0
1	0.52	0	1	0	0.6360	1	0	0
1	0.3	0	1	0	0.3350	0	0	1
1	0.29	1	0	0	0.3140	0	0	1
-1	0.47	0	0	1	0.5940	0	1	0
1	0.39	0	1	0	0.4780	0	1	0
1	0.47	0	0	1	0.5200	0	1	0
-1	0.49	1	0	0	0.5860	0	1	0
-1	0.63	0	0	1	0.6740	0	0	1
-1	0.3	1	0	0	0.3920	1	0	0
-1	0.61	0	0	1	0.6960	0	0	1
-1	0.47	0	0	1	0.5870	0	1	0
1	0.3	0	0	1	0.3450	0	0	1
-1	0.51	0	0	1	0.5800	0	1	0
-1	0.24	1	0	0	0.3880	0	1	0
-1	0.49	1	0	0	0.6450	0	1	0
1	0.66	0	0	1	0.7450	1	0	0
-1	0.65	1	0	0	0.7690	1	0	0
-1	0.46	0	1	0	0.5800	1	0	0
-1	0.45	0	0	1	0.5180	0	1	0
-1	0.47	1	0	0	0.6360	1	0	0
-1	0.29	1	0	0	0.4480	1	0	0
-1	0.57	0	0	1	0.6930	0	0	1
-1	0.2	1	0	0	0.2870	0	0	1
-1	0.35	1	0	0	0.4340	0	1	0
-1	0.61	0	0	1	0.6700	0	0	1
-1	0.31	0	0	1	0.3730	0	1	0
1	0.18	1	0	0	0.2080	0	0	1
1	0.26	0	0	1	0.2920	0	0	1
-1	0.28	1	0	0	0.3640	0	0	1
-1	0.59	0	0	1	0.6940	0	0	1
This entry was posted in Miscellaneous, PyTorch. Bookmark the permalink.