Step 1: Fix the list of categories
Write down the categories the model will choose from, with a plain name and two or three example photos for each. Every photo should belong to exactly one of them: if “bags” and “backpacks” overlap, people will disagree about which is right, and so will the model. If some photos fit none, add an “other” category rather than forcing them into the nearest one.
Then decide what good enough means. Here the goal is to file most photos automatically and be right about 99 times in 100 on those, while a person checks the rest. That’s a better target than overall accuracy, because a model doesn’t have to be sure about every photo to save most of the work.
Step 2: Gather the photos, one folder per category
Put each category’s photos in its own folder, such as photos/mug/, and start each file name with the product it shows (p0042_1.jpg, p0042_2.jpg), because most products have several photos. Your catalog or shop admin can usually export them. A hundred or so products per category is enough to start.
No photos to hand? This script draws sample catalog photos: 160 products of each of six kinds (backpacks, bottles, lamps, mugs, sneakers and t-shirts), each photographed three times. Some are studio shots on a light background and some are lifestyle shots on a colored wall or table, with the product at a different size, place and angle each time, like a real catalog.
CodeOptional: make sample product photos
# pip install pillow numpy
import numpy as np
from pathlib import Path
from PIL import Image, ImageDraw, ImageFilter
rng = np.random.default_rng(7)
SIZE = 128
CATEGORIES = ["backpack", "bottle", "lamp", "mug", "sneaker", "t-shirt"]
def shape(d, kind, x, y, s, color, dark):
"""One product, drawn around (x, y) at scale s."""
if kind == "mug":
d.ellipse([x + 14 * s, y - 14 * s, x + 38 * s, y + 14 * s], outline=color, width=max(2, int(6 * s)))
d.rounded_rectangle([x - 24 * s, y - 28 * s, x + 24 * s, y + 30 * s], 6 * s, fill=color)
elif kind == "bottle":
d.rounded_rectangle([x - 14 * s, y - 20 * s, x + 14 * s, y + 42 * s], 8 * s, fill=color)
d.rectangle([x - 6 * s, y - 36 * s, x + 6 * s, y - 18 * s], fill=color)
d.rectangle([x - 7 * s, y - 44 * s, x + 7 * s, y - 34 * s], fill=dark)
elif kind == "lamp":
d.polygon([(x - 34 * s, y - 2 * s), (x + 34 * s, y - 2 * s), (x + 18 * s, y - 40 * s), (x - 18 * s, y - 40 * s)], fill=color)
d.rectangle([x - 2 * s, y - 2 * s, x + 2 * s, y + 36 * s], fill=dark)
d.ellipse([x - 18 * s, y + 32 * s, x + 18 * s, y + 42 * s], fill=dark)
elif kind == "backpack":
d.rounded_rectangle([x - 28 * s, y - 34 * s, x + 28 * s, y + 38 * s], 14 * s, fill=color)
d.rounded_rectangle([x - 18 * s, y + 6 * s, x + 18 * s, y + 30 * s], 6 * s, fill=dark)
d.arc([x - 10 * s, y - 46 * s, x + 10 * s, y - 26 * s], 180, 360, fill=dark, width=max(2, int(4 * s)))
elif kind == "sneaker":
d.rounded_rectangle([x - 42 * s, y + 10 * s, x + 42 * s, y + 22 * s], 5 * s, fill=(235, 235, 230))
d.polygon([(x - 40 * s, y + 12 * s), (x - 34 * s, y - 14 * s), (x - 8 * s, y - 18 * s), (x + 10 * s, y - 2 * s), (x + 42 * s, y + 4 * s), (x + 40 * s, y + 12 * s)], fill=color)
for k in range(3):
d.line([x - 22 * s + 8 * k * s, y - 14 * s + 4 * k * s, x - 14 * s + 8 * k * s, y - 8 * s + 4 * k * s], fill=dark, width=max(1, int(2 * s)))
else: # t-shirt
d.polygon([(x - 20 * s, y - 34 * s), (x - 44 * s, y - 20 * s), (x - 34 * s, y - 2 * s), (x - 24 * s, y - 8 * s), (x - 24 * s, y + 38 * s),
(x + 24 * s, y + 38 * s), (x + 24 * s, y - 8 * s), (x + 34 * s, y - 2 * s), (x + 44 * s, y - 20 * s), (x + 20 * s, y - 34 * s)], fill=color)
d.ellipse([x - 10 * s, y - 40 * s, x + 10 * s, y - 26 * s], fill=dark)
def photo(kind, color):
"""A catalog photo: a plain or lifestyle background, the product at a random size and place, turned a little."""
if rng.random() < 0.55: # studio: light grey to white
bg = np.full((SIZE, SIZE, 3), rng.uniform(220, 250), dtype="float32") + rng.normal(0, 2, (SIZE, SIZE, 1))
else: # lifestyle: a colored wall or a wooden table
bg = np.ones((SIZE, SIZE, 3), dtype="float32") * rng.uniform(90, 200, 3) + rng.normal(0, 6, (SIZE, SIZE, 1))
img = Image.fromarray(bg.clip(0, 255).astype("uint8"))
layer = Image.new("RGBA", (SIZE, SIZE))
d = ImageDraw.Draw(layer)
x, y, s = SIZE / 2 + rng.uniform(-22, 22), SIZE / 2 + rng.uniform(-16, 16), rng.uniform(0.6, 1.3)
d.ellipse([x - 40 * s, y + 38 * s, x + 40 * s, y + 48 * s], fill=(0, 0, 0, 40)) # a soft shadow
light = rng.uniform(0.8, 1.15)
c = tuple(int(min(255, v * light)) for v in color)
shape(d, kind, x, y, s, c, tuple(int(v * 0.55) for v in c))
layer = layer.rotate(rng.uniform(-20, 20), center=(x, y))
img.paste(layer, mask=layer)
return img.filter(ImageFilter.GaussianBlur(rng.uniform(0.2, 0.9)))
# 160 products of each kind, each photographed 3 times: photos/mug/p0042_1.jpg, ...
n = 0
for kind in CATEGORIES:
folder = Path("photos") / kind
folder.mkdir(parents=True, exist_ok=True)
for _ in range(160):
n += 1
color = tuple(int(v) for v in rng.uniform(30, 230, 3))
for shot in (1, 2, 3):
photo(kind, color).save(folder / f"p{n:04d}_{shot}.jpg", quality=90)
print({f.name: len(list(f.glob("*.jpg"))) for f in sorted(Path("photos").iterdir())})Step 3: Keep every photo of a product on the same side
Split the photos three ways: training (to learn from), validation (to make choices on, such as the confidence for automatic filing) and test (to check the result once, at the end). Split by product, not by photo. If one photo of a red mug is in training and another in test, the model only has to recognize that mug, and the test score is better than it will be on new products. That’s data leakage.
CodeSplit by product
# pip install scikit-learn pandas
import pandas as pd
from sklearn.model_selection import GroupShuffleSplit
photos = pd.DataFrame({"path": sorted(Path("photos").glob("*/*.jpg"))})
photos["category"] = photos.path.map(lambda p: p.parent.name)
photos["product"] = photos.path.map(lambda p: p.stem.split("_")[0]) # p0042_1.jpg -> p0042
# 70% of products to learn from, 15% to make choices on, 15% to test once at the end
outer = GroupShuffleSplit(n_splits=1, test_size=0.3, random_state=7)
train_idx, rest_idx = next(outer.split(photos, groups=photos["product"]))
train, rest = photos.iloc[train_idx], photos.iloc[rest_idx]
inner = GroupShuffleSplit(n_splits=1, test_size=0.5, random_state=7)
val_idx, test_idx = next(inner.split(rest, groups=rest["product"]))
val, test = rest.iloc[val_idx], rest.iloc[test_idx]
assert not set(train["product"]) & set(test["product"]), "a product is on both sides"
print({name: len(part) for name, part in [("train", train), ("validation", val), ("test", test)]})That gives 2,016 training photos, 432 for validation and 432 for the test, and the check at the end confirms no product is on both sides.
Step 4: Start with a simple baseline
Before training anything complicated, see how far a simple baseline gets. This one shrinks each photo to a 16 × 16 grayscale thumbnail, counts its colors, and trains a logistic regression on those numbers. It runs in seconds.
CodeA baseline from thumbnails and colors
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline
def simple_features(path):
"""A 16 x 16 grayscale thumbnail plus a color histogram: the outline and the colors, nothing more."""
img = Image.open(path).convert("RGB")
thumb = np.asarray(img.convert("L").resize((16, 16)), dtype="float32").ravel() / 255
hist = np.concatenate([np.histogram(np.asarray(img)[..., c], bins=8, range=(0, 255))[0] for c in range(3)]) / SIZE ** 2
return np.concatenate([thumb, hist])
X_train = np.stack(train.path.map(simple_features))
X_val = np.stack(val.path.map(simple_features))
baseline = make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000)).fit(X_train, train.category)
print(f"Baseline accuracy on validation photos: {(baseline.predict(X_val) == val.category).mean():.1%}")It gets 49.5% of the validation photos right: better than guessing (1 in 6, about 17%), but far from useful. The reason is that it compares photos pixel by pixel, and the same mug moves, grows and turns from photo to photo, so its pixels land in different places. A model for photos needs to find shapes wherever they are.
Step 5: Train a small network that finds shapes anywhere in the photo
A convolutional network slides small filters over the photo, so it finds a handle or a sole wherever it is. Each block finds larger patterns than the one before: edges, then parts, then whole objects. Data augmentation flips, turns, zooms and relights the training photos a little on every pass, so the model learns the product, not the exact photo. Training stops once the validation score has stopped improving for four rounds, and keeps the best version.
CodeA small network with augmentation
# pip install tensorflow
from tensorflow import keras
keras.utils.set_random_seed(7)
IMG = 64
def load(paths):
return np.stack([np.asarray(Image.open(p).convert("RGB").resize((IMG, IMG)), dtype="float32") for p in paths])
classes = sorted(photos.category.unique())
to_id = {c: i for i, c in enumerate(classes)}
x_train, y_train = load(train.path), train.category.map(to_id).values
x_val, y_val = load(val.path), val.category.map(to_id).values
def block(filters):
"""Patterns at one scale (edges, then parts, then whole shapes), wherever they are in the photo."""
return [keras.layers.Conv2D(filters, 3, padding="same", use_bias=False),
keras.layers.BatchNormalization(),
keras.layers.Activation("relu"),
keras.layers.MaxPooling2D()]
model = keras.Sequential([
keras.Input((IMG, IMG, 3)),
keras.layers.Rescaling(1 / 255),
# Augmentation: each pass over the photos sees them flipped, turned, zoomed and lit a little differently
keras.layers.RandomFlip("horizontal"),
keras.layers.RandomRotation(0.05),
keras.layers.RandomZoom(0.15),
keras.layers.RandomContrast(0.2),
*block(32), *block(64), *block(128),
keras.layers.GlobalAveragePooling2D(),
keras.layers.Dropout(0.3),
keras.layers.Dense(len(classes), activation="softmax"),
])
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
stop = keras.callbacks.EarlyStopping(patience=4, restore_best_weights=True)
model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=25, batch_size=32, callbacks=[stop], verbose=2)
print(f"Network accuracy on validation photos: {model.evaluate(x_val, y_val, verbose=0)[1]:.1%}")On a laptop processor this trains in about a minute and a half and gets 97.0% of the validation photos right. Your numbers may differ by a point or so, because training starts from random values and computers round slightly differently.
Step 6: Read the confusion matrix
Now check the model once on the test photos it has never seen. Overall accuracy hides which categories it mixes up; the confusion matrix shows them. Each row is the real category and each column what the model said, so everything off the diagonal is a mistake.
CodeTest accuracy, the confusion matrix and each category’s scores
from sklearn.metrics import classification_report, confusion_matrix
x_test, y_test = load(test.path), test.category.map(to_id).values
probs = model.predict(x_test, verbose=0)
pred, conf = probs.argmax(axis=1), probs.max(axis=1) # the likeliest category, and how sure
print(f"Test accuracy: {(pred == y_test).mean():.1%}\n")
print(pd.DataFrame(confusion_matrix(y_test, pred), index=classes, columns=classes)) # rows: the real category
print(classification_report(y_test, pred, target_names=classes, digits=3))The model gets 95.4% of the test photos right. Its main confusion is calling things t-shirts: six backpacks and five mugs were filed as t-shirts. So when it says “t-shirt”, it’s right 85.4% of the time (that’s precision), while it finds 97.4% of the real t-shirts (that’s recall). Sneakers are never missed. A pattern like this tells you where to look first.
Step 7: Look at every mistake
This copies each wrong test photo into a folder named after the mix-up, with the model’s confidence at the start of the file name, so you can open the folders and look.
CodeCollect the mistakes to look at
import shutil
# Copy every wrongly sorted test photo into mistakes/, in a folder named "real category_as_what the model said"
shutil.rmtree("mistakes", ignore_errors=True)
for path, real, said, c in zip(test.path, test.category, pred, conf):
if classes[said] != real:
folder = Path("mistakes") / f"{real}_as_{classes[said]}"
folder.mkdir(parents=True, exist_ok=True)
shutil.copy(path, folder / f"{c:.2f}_{path.name}") # the file name starts with the confidence
for folder in sorted(Path("mistakes").iterdir()):
print(folder.name, sorted(f.name[:4] for f in folder.iterdir()))Most of the 20 mistakes are photos a person would look at twice, too: a pale product on an almost white background, a product cut off at the edge of the frame, or one so close that it fills the photo. And 16 of the 20 were made with less than 0.6 confidence. Both are useful. The first tells you how to take better photos (a contrasting background, the whole product in frame) or which photos to add more of; the second means the model knows when it’s unsure, which the next step uses.
Step 8: File confident photos, review the rest
Use the validation photos to find the lowest confidence at which the photos the model files automatically are right 99% of the time, then check that confidence once on the test photos.
CodeChoose the confidence for automatic filing
# Choose the confidence on the validation photos: the lowest at which auto-filed photos are 99% right
val_probs = model.predict(x_val, verbose=0)
val_conf, val_ok = val_probs.max(axis=1), val_probs.argmax(axis=1) == y_val
for t in np.arange(0.50, 1.0, 0.01):
if val_ok[val_conf >= t].mean() >= 0.99:
break
print(f"File automatically from {t:.2f} confidence")
# Then check it once on the test photos
auto = conf >= t
print(f"Filed automatically: {auto.mean():.0%} of test photos, {(pred[auto] == y_test[auto]).mean():.1%} of them right")
print(f"Sent for review: {(~auto).sum()} photos")From 0.82 confidence, the model files 79% of the test photos automatically, and every one of those is right. The other 91 photos (21%) go to a person, who gets the model’s top three suggestions, so most reviews take one click. Raise the confidence to send more to review and make even fewer mistakes; lower it to save more time. Choose it on the validation photos, never on the test photos.
Step 9: Sort new photos as they arrive
CodeSuggest a category for a new photo
def suggest(path, threshold=t):
"""The likeliest category for a new photo, how sure the model is, and the top three for a reviewer."""
p = model.predict(load([path]), verbose=0)[0]
top = p.argsort()[::-1][:3]
return {"category": classes[top[0]],
"confidence": round(float(p[top[0]]), 3),
"file_automatically": bool(p[top[0]] >= threshold),
"suggestions": [(classes[i], round(float(p[i]), 3)) for i in top]}
print(suggest(test.path.iloc[0]))
model.save("product_photos.keras") # load it later with keras.models.load_model("product_photos.keras")- Run
suggest()when a photo is uploaded, file it iffile_automaticallyis true, and put the rest in a review list with the suggestions. - Keep the reviewers’ answers. Every reviewed photo is a new, correctly labeled example, often of exactly the kind the model finds hard. Add them to the training photos and retrain every month or two.
- Watch the share sent for review. If it climbs, the photos have changed: a new photographer, a new background or a new kind of product. That’s data drift, and retraining on recent photos fixes it.
- Adding a category means a new folder of photos and retraining. Until then, those products land in review, which is the safe place for them.
Common pitfalls
- Splitting by photo instead of by product, so the test score is too good to be true.
- Categories that overlap, so even people disagree about the right answer.
- Judging the model by overall accuracy, which hides that one category catches most of the mistakes.
- Choosing the confidence for automatic filing on the test photos, so the test score looks better than real use.
- Never retraining, so new products and new photo styles slowly drift into the wrong folders.