thepowerfuldeez/sd21-controlnet-canny

Converted Canny SD 2.1-base model from https://huggingface.co/thibaud/controlnet-sd21/ to diffusers format.

Saved only ControlNet weights

Usage:

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, DEISMultistepScheduler
import cv2
from PIL import Image
import numpy as np

pipe = StableDiffusionControlNetPipeline.from_pretrained(
    "stabilityai/stable-diffusion-2-1-base",
    safety_checker=None,
    # revision='fp16',
    # torch_dtype=torch.float16,
    controlnet=ControlNetModel.from_pretrained("thepowefuldeez/sd21-controlnet-canny")
).to('cuda')
pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config)

image = np.array(Image.open("10.png"))

low_threshold = 100
high_threshold = 200

image = cv2.Canny(image, low_threshold, high_threshold)
image = image[:, :, None]
image = np.concatenate([image, image, image], axis=2)
canny_image = Image.fromarray(image)

im = pipe(
    "beautiful woman", image=canny_image, num_inference_steps=30, 
    negative_prompt="ugly, blurry, bad, deformed, bad anatomy", 
    generator=torch.manual_seed(42)
).images[0]