HiDiffusion

HiDiffusion is compatible with various specialized models. You can wrap any standard diffusers pipeline with apply_hidiffusion to enable high-resolution generation.

Ghibli-Diffusion Example

from diffusers import StableDiffusionPipeline
from hidiffusion import apply_hidiffusion

model_id = "nitrosocke/Ghibli-Diffusion"
pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda")
apply_hidiffusion(pipe)

image = pipe("ghibli style magical princess with golden hair", height=1024, width=1024, eta=1.0).images[0]

Playground Example

from diffusers import DiffusionPipeline
from hidiffusion import apply_hidiffusion

pipe = DiffusionPipeline.from_pretrained(
    "playgroundai/playground-v2-1024px-aesthetic",
    torch_dtype=torch.float16,
    use_safetensors=True,
    add_watermarker=False,
    variant="fp16"
).to("cuda")

pipe.enable_xformers_memory_efficient_attention()
apply_hidiffusion(pipe)

# Note: For Playground, you may need to adjust guidance_scale (e.g., 3.0 to 5.0)
image = pipe(
    prompt="The little girl riding a bike...", 
    guidance_scale=3.0, 
    height=2048, 
    width=2048, 
    negative_prompt="blurry, ugly, duplicate, poorly drawn, deformed, mosaic"
).images[0]

To use HiDiffusion with Stable Diffusion XL, initialize a StableDiffusionXLPipeline and call apply_hidiffusion(pipe). You can generate high-resolution images (e.g., 2048x2048 or even 4096x4096) by setting the height and width parameters in the pipeline call.

from hidiffusion import apply_hidiffusion, remove_hidiffusion
from diffusers import StableDiffusionXLPipeline, DDIMScheduler
import torch

pretrain_model = "stabilityai/stable-diffusion-xl-base-1.0"
scheduler = DDIMScheduler.from_pretrained(pretrain_model, subfolder="scheduler")
pipe = StableDiffusionXLPipeline.from_pretrained(pretrain_model, scheduler = scheduler, torch_dtype=torch.float16, variant="fp16").to("cuda")

# Apply hidiffusion with a single line of code.
apply_hidiffusion(pipe)

prompt = "Standing tall amidst the ruins, a stone golem awakens, vines and flowers sprouting from the crevices in its body."
negative_prompt = "blurry, ugly, duplicate, poorly drawn face, deformed, mosaic, artifacts, bad limbs"
image = pipe(prompt, guidance_scale=7.5, height=2048, width=2048, eta=1.0, negative_prompt=negative_prompt).images[0]
image.save(f"golem.jpg")

Use StableDiffusionXLControlNetImg2ImgPipeline combined with HiDiffusion to transform an existing image while maintaining structural control via ControlNet.

Note: When using high resolutions, ensure you set height and width appropriately in the pipeline call.

from diffusers import ControlNetModel, StableDiffusionXLControlNetImg2ImgPipeline, DDIMScheduler
import torch
from PIL import Image
from hidiffusion import apply_hidiffusion

# ... (setup controlnet and scheduler) ...

pipe = StableDiffusionXLControlNetImg2ImgPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    controlnet=controlnet,
    scheduler=scheduler,
    torch_dtype=torch.float16,
).to("cuda")

apply_hidiffusion(pipe)

# ... (prepare ori_image and canny_image) ...

image = pipe(
    prompt="Lara Croft with brown hair...",
    image=ori_image,
    control_image=canny_image,
    height=1536,
    width=2048,
    strength=0.99,
    num_inference_steps=50,
    controlnet_conditioning_scale=0.5,
    guidance_scale=12.5,
    negative_prompt="underexposed, poorly drawn hands...",
    eta=1.0
).images[0]

image.save("lara.jpg")

Use DiffusionPipeline with the runwayml/stable-diffusion-v1-5 model. HiDiffusion enables high-resolution generation (e.g., 2048x2048) for this version as well.

from hidiffusion import apply_hidiffusion, remove_hidiffusion
from diffusers import DiffusionPipeline, DDIMScheduler
import torch

pretrain_model = "runwayml/stable-diffusion-v1-5"
scheduler = DDIMScheduler.from_pretrained(pretrain_model, subfolder="scheduler")
pipe = DiffusionPipeline.from_pretrained(pretrain_model, scheduler = scheduler, torch_dtype=torch.float16).to("cuda")

# Apply hidiffusion with a single line of code.
apply_hidiffusion(pipe)

prompt = "thick strokes, bright colors, an exotic fox, cute, chibi kawaii. detailed fur, hyperdetailed , big reflective eyes, fairytale, artstation,centered composition, perfect composition, centered, vibrant colors, muted colors, high detailed, 8k."
negative_prompt = "ugly, tiling, poorly drawn face, out of frame, disfigured, deformed, blurry, bad anatomy, blurred."
image = pipe(prompt, guidance_scale=7.5, height=1024, width=1024, eta=1.0, negative_prompt=negative_prompt).images[0]
image.save(f"fox.jpg")

HiDiffusion can be applied to AutoPipelineForInpainting to generate high-resolution details within a masked area of an image.

Usage:

1. Load an inpainting pipeline (e.g., diffusers/stable-diffusion-xl-1.0-inpainting-0.1).
2. Apply apply_hidiffusion(pipeline).
3. Provide the image (base image) and mask_image to the pipeline call along with high height and width values.

import torch
from diffusers import AutoPipelineForInpainting, DDIMScheduler
from diffusers.utils import load_image
from hidiffusion import apply_hidiffusion
from PIL import Image

scheduler = DDIMScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler")
pipeline = AutoPipelineForInpainting.from_pretrained(
    "diffusers/stable-diffusion-xl-1.0-inpainting-0.1", 
    torch_dtype=torch.float16, 
    variant="fp16", 
    scheduler=scheduler
)

apply_hidiffusion(pipeline)

pipeline.enable_model_cpu_offload()
pipeline.enable_xformers_memory_efficient_attention()

img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"
init_image = load_image(img_url)
mask_image = Image.open("./assets/mask_image.png")

image = pipeline(
    prompt="A steampunk explorer...",
    image=init_image, 
    mask_image=mask_image, 
    height=2048, 
    width=2048, 
    strength=0.85, 
    guidance_scale=12.5, 
    negative_prompt="blurry, ugly, duplicate, poorly drawn, deformed, mosaic",
    eta=1.0
).images[0]

image.save('steampunk_explorer.jpg')

Use DiffusionPipeline with the stabilityai/stable-diffusion-2-1-base model. Applying HiDiffusion allows for higher resolutions like 2048x2048.

from hidiffusion import apply_hidiffusion, remove_hidiffusion
from diffusers import DiffusionPipeline, DDIMScheduler
import torch

pretrain_model = "stabilityai/stable-diffusion-2-1-base"
scheduler = DDIMScheduler.from_pretrained(pretrain_model, subfolder="scheduler")
pipe = DiffusionPipeline.from_pretrained(pretrain_model, scheduler = scheduler, torch_dtype=torch.float16).to("cuda")

# Apply hidiffusion with a single line of code.
apply_hidiffusion(pipe)

prompt = "An adorable happy brown border collie sitting on a bed, high detail."
negative_prompt = "ugly, tiling, out of frame, poorly drawn face, extra limbs, disfigured, deformed, body out of frame, blurry, bad anatomy, blurred, artifacts, bad proportions."
image = pipe(prompt, guidance_scale=7.5, height=1024, width=1024, eta=1.0, negative_prompt=negative_prompt).images[0]
image.save(f"collie.jpg")

You can use HiDiffusion with StableDiffusionXLControlNetPipeline to generate high-resolution images guided by a ControlNet (e.g., Canny edge detection).

Key steps:

1. Initialize the ControlNetModel and StableDiffusionXLControlNetPipeline.
2. Call apply_hidiffusion(pipe).
3. Pass parameters like height and width (e.g., 2048x2048) to the pipeline to leverage high-resolution capabilities.

from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, DDIMScheduler
import torch
from PIL import Image
import cv2
import numpy as np
from hidiffusion import apply_hidiffusion

# ... (setup controlnet and scheduler) ...

pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0", 
    controlnet=controlnet, 
    torch_dtype=torch.float16,
    scheduler=scheduler
)

# Apply hidiffusion
apply_hidiffusion(pipe)

# Generate high-res image
image = pipe(
    prompt="The Joker, high face detail, high detail, muted color, 8k",
    controlnet_conditioning_scale=0.5,
    image=canny_image,
    height=2048,
    width=2048,
    guidance_scale=7.5,
    negative_prompt="blurry, ugly, duplicate, poorly drawn, deformed, mosaic.",
    eta=1.0
).images[0]

image.save('joker.jpg')