wyysf commited on
Commit
c459ff0
β€’
1 Parent(s): 82ba93f
.gitattributes CHANGED
@@ -38,3 +38,4 @@ apps/third_party/Wonder3D/assets/fig_teaser.png filter=lfs diff=lfs merge=lfs -t
38
  asset/video_cover.png filter=lfs diff=lfs merge=lfs -text
39
  apps/InstantMeshes filter=lfs diff=lfs merge=lfs -text
40
  apps/third_party/InstantMeshes filter=lfs diff=lfs merge=lfs -text
 
 
38
  asset/video_cover.png filter=lfs diff=lfs merge=lfs -text
39
  apps/InstantMeshes filter=lfs diff=lfs merge=lfs -text
40
  apps/third_party/InstantMeshes filter=lfs diff=lfs merge=lfs -text
41
+ apps/third_party/quadriflow filter=lfs diff=lfs merge=lfs -text
apps/__pycache__/mv_models.cpython-38.pyc CHANGED
Binary files a/apps/__pycache__/mv_models.cpython-38.pyc and b/apps/__pycache__/mv_models.cpython-38.pyc differ
 
apps/__pycache__/utils.cpython-38.pyc CHANGED
Binary files a/apps/__pycache__/utils.cpython-38.pyc and b/apps/__pycache__/utils.cpython-38.pyc differ
 
apps/gradio_app.py CHANGED
@@ -20,6 +20,8 @@ from utils import *
20
  proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21
  sys.path.append(os.path.join(proj_dir))
22
 
 
 
23
  import craftsman
24
  from craftsman.systems.base import BaseSystem
25
  from craftsman.utils.config import ExperimentConfig, load_config
@@ -104,16 +106,21 @@ def image2mesh(view_front: np.ndarray,
104
  )
105
  assert len(mesh_outputs) == 1, "Only support single mesh output for gradio demo"
106
  mesh = trimesh.Trimesh(mesh_outputs[0][0], mesh_outputs[0][1])
107
- filepath = f"{cached_dir}/{time.time()}.obj"
 
108
  mesh.export(filepath, include_normals=True)
109
 
110
  if 'Remesh' in more:
 
111
  print("Remeshing with Instant Meshes...")
112
  target_face_count = int(len(mesh.faces)/10)
113
  # command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 6 -p 6 -o {filepath.replace('.obj', '_remeshed.obj')}"
114
- command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 4 -p 4 -o {filepath.replace('.obj', '_remeshed.obj')}"
 
 
115
  os.system(command)
116
- filepath = filepath.replace('.obj', '_remeshed.obj')
 
117
 
118
  return filepath
119
 
 
20
  proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21
  sys.path.append(os.path.join(proj_dir))
22
 
23
+ import tempfile
24
+
25
  import craftsman
26
  from craftsman.systems.base import BaseSystem
27
  from craftsman.utils.config import ExperimentConfig, load_config
 
106
  )
107
  assert len(mesh_outputs) == 1, "Only support single mesh output for gradio demo"
108
  mesh = trimesh.Trimesh(mesh_outputs[0][0], mesh_outputs[0][1])
109
+ # filepath = f"{cached_dir}/{time.time()}.obj"
110
+ filepath = tempfile.NamedTemporaryFile(suffix=f".obj", delete=False).name
111
  mesh.export(filepath, include_normals=True)
112
 
113
  if 'Remesh' in more:
114
+ remeshed_filepath = tempfile.NamedTemporaryFile(suffix=f"_remeshed.obj", delete=False).name
115
  print("Remeshing with Instant Meshes...")
116
  target_face_count = int(len(mesh.faces)/10)
117
  # command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 6 -p 6 -o {filepath.replace('.obj', '_remeshed.obj')}"
118
+ # command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 4 -p 4 -o {filepath.replace('.obj', '_remeshed.obj')}"
119
+ # command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -o {filepath.replace('.obj', '_remeshed.obj')}"
120
+ command = f"{proj_dir}/apps/third_party/quadriflow -i {filepath} -f {target_face_count} -o {remeshed_filepath}"
121
  os.system(command)
122
+ filepath = remeshed_filepath
123
+ # filepath = filepath.replace('.obj', '_remeshed.obj')
124
 
125
  return filepath
126
 
apps/mv_models.py CHANGED
@@ -19,6 +19,37 @@ from huggingface_hub import hf_hub_download
19
 
20
  parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  class GenMVImage(object):
23
  def __init__(self, device):
24
  self.seed = 1024
@@ -96,19 +127,19 @@ class GenMVImage(object):
96
  return mv_imgs[1], mv_imgs[2], mv_imgs[3], mv_imgs[0]
97
 
98
  def gen_image_from_wonder3d(self, image, crop_size):
99
- sys.path.append(f"{parent_dir}/apps/third_party/Wonder3D")
100
- from .third_party.Wonder3D.mvdiffusion.pipelines.pipeline_mvdiffusion_image import MVDiffusionImagePipeline
101
  weight_dtype = torch.float16
102
  batch = prepare_data(image, crop_size)
103
 
104
  if "wonder3d" in self.pipelines.keys():
105
  pipeline = self.pipelines['wonder3d']
106
  else:
107
- self.pipelines['wonder3d'] = MVDiffusionImagePipeline.from_pretrained(
108
- 'flamehaze1115/wonder3d-v1.0',
109
- custom_pipeline=f'{parent_dir}/apps/third_party/Wonder3D/mvdiffusion/pipelines/pipeline_mvdiffusion_image.py',
110
- torch_dtype=weight_dtype
111
- )
112
  self.pipelines['wonder3d'].unet.enable_xformers_memory_efficient_attention()
113
  self.pipelines['wonder3d'].to(self.device)
114
  self.pipelines['wonder3d'].set_progress_bar_config(disable=True)
 
19
 
20
  parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21
 
22
+
23
+ @dataclass
24
+ class TestConfig:
25
+ pretrained_model_name_or_path: str
26
+ pretrained_unet_path: str
27
+ revision: Optional[str]
28
+ validation_dataset: Dict
29
+ save_dir: str
30
+ seed: Optional[int]
31
+ validation_batch_size: int
32
+ dataloader_num_workers: int
33
+
34
+ local_rank: int
35
+
36
+ pipe_kwargs: Dict
37
+ pipe_validation_kwargs: Dict
38
+ unet_from_pretrained_kwargs: Dict
39
+ validation_guidance_scales: List[float]
40
+ validation_grid_nrow: int
41
+ camera_embedding_lr_mult: float
42
+
43
+ num_views: int
44
+ camera_embedding_type: str
45
+
46
+ pred_type: str # joint, or ablation
47
+
48
+ enable_xformers_memory_efficient_attention: bool
49
+
50
+ cond_on_normals: bool
51
+ cond_on_colors: bool
52
+
53
  class GenMVImage(object):
54
  def __init__(self, device):
55
  self.seed = 1024
 
127
  return mv_imgs[1], mv_imgs[2], mv_imgs[3], mv_imgs[0]
128
 
129
  def gen_image_from_wonder3d(self, image, crop_size):
130
+ from diffusers import DiffusionPipeline # only tested on diffusers[torch]==0.19.3, may have conflicts with newer versions of diffusers
131
+
132
  weight_dtype = torch.float16
133
  batch = prepare_data(image, crop_size)
134
 
135
  if "wonder3d" in self.pipelines.keys():
136
  pipeline = self.pipelines['wonder3d']
137
  else:
138
+ self.pipelines['wonder3d'] = DiffusionPipeline.from_pretrained(
139
+ 'flamehaze1115/wonder3d-v1.0', # or use local checkpoint './ckpts'
140
+ custom_pipeline='flamehaze1115/wonder3d-pipeline',
141
+ torch_dtype=torch.float16
142
+ )
143
  self.pipelines['wonder3d'].unet.enable_xformers_memory_efficient_attention()
144
  self.pipelines['wonder3d'].to(self.device)
145
  self.pipelines['wonder3d'].set_progress_bar_config(disable=True)
apps/third_party/CRM/imagedream/ldm/util.py CHANGED
@@ -95,9 +95,9 @@ def get_obj_from_str(string, reload=False):
95
  importlib.reload(module_imp)
96
 
97
  if 'imagedream' in module:
98
- module = 'third_party.CRM.'+module
99
  if 'lib' in module:
100
- module = 'third_party.CRM.'+module
101
  return getattr(importlib.import_module(module, package=None), cls)
102
 
103
 
 
95
  importlib.reload(module_imp)
96
 
97
  if 'imagedream' in module:
98
+ module = 'apps.third_party.CRM.'+module
99
  if 'lib' in module:
100
+ module = 'apps.third_party.CRM.'+module
101
  return getattr(importlib.import_module(module, package=None), cls)
102
 
103
 
apps/third_party/CRM/libs/sample.py CHANGED
@@ -6,7 +6,7 @@ from imagedream.ldm.util import set_seed, add_random_background
6
  # import sys
7
  # proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
8
  # sys.path.append(proj_dir)
9
- from third_party.CRM.libs.base_utils import do_resize_content
10
  from imagedream.ldm.models.diffusion.ddim import DDIMSampler
11
  from torchvision import transforms as T
12
 
 
6
  # import sys
7
  # proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
8
  # sys.path.append(proj_dir)
9
+ from apps.third_party.CRM.libs.base_utils import do_resize_content
10
  from imagedream.ldm.models.diffusion.ddim import DDIMSampler
11
  from torchvision import transforms as T
12
 
apps/third_party/InstantMeshes CHANGED
File without changes
apps/third_party/Wonder3D/mvdiffusion/data/single_image_dataset.py CHANGED
@@ -107,7 +107,7 @@ class SingleImageDataset(Dataset):
107
  elif self.num_views == 6:
108
  self.view_types = ['front', 'front_right', 'right', 'back', 'left', 'front_left']
109
 
110
- self.fix_cam_pose_dir = "./third_party/Wonder3D/mvdiffusion/data/fixed_poses/nine_views"
111
 
112
  self.fix_cam_poses = self.load_fixed_poses() # world2cam matrix
113
 
 
107
  elif self.num_views == 6:
108
  self.view_types = ['front', 'front_right', 'right', 'back', 'left', 'front_left']
109
 
110
+ self.fix_cam_pose_dir = "apps/third_party/Wonder3D/mvdiffusion/data/fixed_poses/nine_views"
111
 
112
  self.fix_cam_poses = self.load_fixed_poses() # world2cam matrix
113
 
apps/utils.py CHANGED
@@ -17,6 +17,7 @@ rembg_session = rembg.new_session()
17
  from segment_anything import sam_model_registry, SamPredictor
18
 
19
  import craftsman
 
20
  from craftsman.utils.config import ExperimentConfig, load_config
21
 
22
  parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -47,7 +48,7 @@ def load_model(
47
  )
48
 
49
  print(f"Restoring states from the checkpoint path at {ckpt_path} with config {cfg}")
50
- system.load_state_dict(torch.load(ckpt_path)['state_dict'])
51
  system = system.to(device).eval()
52
 
53
  return system
@@ -135,7 +136,7 @@ def save_image(tensor):
135
  return ndarr
136
 
137
  def prepare_data(single_image, crop_size):
138
- from third_party.Wonder3D.mvdiffusion.data.single_image_dataset import SingleImageDataset
139
  dataset = SingleImageDataset(root_dir='', num_views=6, img_wh=[256, 256], bg_color='white', crop_size=crop_size, single_image=single_image)
140
  return dataset[0]
141
 
 
17
  from segment_anything import sam_model_registry, SamPredictor
18
 
19
  import craftsman
20
+ from craftsman.systems.base import BaseSystem
21
  from craftsman.utils.config import ExperimentConfig, load_config
22
 
23
  parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
48
  )
49
 
50
  print(f"Restoring states from the checkpoint path at {ckpt_path} with config {cfg}")
51
+ system.load_state_dict(torch.load(ckpt_path, map_location=torch.device('cpu'))['state_dict'])
52
  system = system.to(device).eval()
53
 
54
  return system
 
136
  return ndarr
137
 
138
  def prepare_data(single_image, crop_size):
139
+ from apps.third_party.Wonder3D.mvdiffusion.data.single_image_dataset import SingleImageDataset
140
  dataset = SingleImageDataset(root_dir='', num_views=6, img_wh=[256, 256], bg_color='white', crop_size=crop_size, single_image=single_image)
141
  return dataset[0]
142
 
gradio_app.py CHANGED
@@ -19,6 +19,7 @@ import gradio as gr
19
  proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
20
  sys.path.append(os.path.join(proj_dir))
21
 
 
22
  import craftsman
23
  from craftsman.systems.base import BaseSystem
24
  from craftsman.utils.config import ExperimentConfig, load_config
@@ -104,18 +105,22 @@ def image2mesh(view_front: np.ndarray,
104
  )
105
  assert len(mesh_outputs) == 1, "Only support single mesh output for gradio demo"
106
  mesh = trimesh.Trimesh(mesh_outputs[0][0], mesh_outputs[0][1])
107
- filepath = f"{cached_dir}/{time.time()}.obj"
 
108
  mesh.export(filepath, include_normals=True)
109
 
110
  if 'Remesh' in more:
 
111
  print("Remeshing with Instant Meshes...")
112
- target_face_count = int(len(mesh.faces)/10)
113
- command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -d -S 0 -r 6 -p 6 -o {filepath.replace('.obj', '_remeshed.obj')}"
 
114
  os.system(command)
115
- filepath = filepath.replace('.obj', '_remeshed.obj')
 
116
 
117
  return filepath
118
-
119
  if __name__=="__main__":
120
  parser = argparse.ArgumentParser()
121
  # parser.add_argument("--model_path", type=str, required=True, help="Path to the object file",)
@@ -201,10 +206,10 @@ if __name__=="__main__":
201
  view_back = gr.Image(label="Back", interactive=True, show_label=True)
202
  view_left = gr.Image(label="Left", interactive=True, show_label=True)
203
 
204
- with gr.Accordion('Advanced options', open=False):
205
- with gr.Row(equal_height=True):
206
- run_mv_btn = gr.Button('Only Generate 2D', interactive=True)
207
- run_3d_btn = gr.Button('Only Generate 3D', interactive=True)
208
 
209
  with gr.Accordion('Advanced options (2D)', open=False):
210
  with gr.Row():
 
19
  proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
20
  sys.path.append(os.path.join(proj_dir))
21
 
22
+ import tempfile
23
  import craftsman
24
  from craftsman.systems.base import BaseSystem
25
  from craftsman.utils.config import ExperimentConfig, load_config
 
105
  )
106
  assert len(mesh_outputs) == 1, "Only support single mesh output for gradio demo"
107
  mesh = trimesh.Trimesh(mesh_outputs[0][0], mesh_outputs[0][1])
108
+ # filepath = f"{cached_dir}/{time.time()}.obj"
109
+ filepath = tempfile.NamedTemporaryFile(suffix=f".obj", delete=False).name
110
  mesh.export(filepath, include_normals=True)
111
 
112
  if 'Remesh' in more:
113
+ remeshed_filepath = tempfile.NamedTemporaryFile(suffix=f"_remeshed.obj", delete=False).name
114
  print("Remeshing with Instant Meshes...")
115
+ # target_face_count = int(len(mesh.faces)/10)
116
+ target_face_count = 1000
117
+ command = f"{proj_dir}/apps/third_party/InstantMeshes {filepath} -f {target_face_count} -o {remeshed_filepath}"
118
  os.system(command)
119
+ filepath = remeshed_filepath
120
+ # filepath = filepath.replace('.obj', '_remeshed.obj')
121
 
122
  return filepath
123
+
124
  if __name__=="__main__":
125
  parser = argparse.ArgumentParser()
126
  # parser.add_argument("--model_path", type=str, required=True, help="Path to the object file",)
 
206
  view_back = gr.Image(label="Back", interactive=True, show_label=True)
207
  view_left = gr.Image(label="Left", interactive=True, show_label=True)
208
 
209
+ # with gr.Accordion('Advanced options', open=False):
210
+ with gr.Row(equal_height=True):
211
+ run_mv_btn = gr.Button('Only Generate 2D', interactive=True)
212
+ run_3d_btn = gr.Button('Only Generate 3D', interactive=True)
213
 
214
  with gr.Accordion('Advanced options (2D)', open=False):
215
  with gr.Row():
requirements.txt CHANGED
@@ -1,5 +1,5 @@
1
  datasets==2.19.0
2
- diffusers==0.27.2
3
  einops==0.8.0
4
  huggingface-hub==0.22.2
5
  imageio==2.34.1
 
1
  datasets==2.19.0
2
+ diffusers==0.19.3
3
  einops==0.8.0
4
  huggingface-hub==0.22.2
5
  imageio==2.34.1