feat: final

README.md

@@ -0,0 +1,75 @@
+# How to run
+
+## DreamGaussion & MVDream
+
+Please install PyTorch & Nvidia CUDA toolkit first.
+
+I run my code on PyTorch 2.1.0 with 11.8 CUDA.
+
+
+And run these code to build the environment to generate 3D Pokemon models.
+```
+git clone https://github.com/dreamgaussian/dreamgaussian.git
+
+cd dreamgaussian
+
+# a modified gaussian splatting (+ depth, alpha rendering)
+git clone --recursive https://github.com/ashawkey/diff-gaussian-rasterization
+pip install ./diff-gaussian-rasterization
+
+# simple-knn
+pip install ./simple-knn
+
+# nvdiffrast
+pip install git+https://github.com/NVlabs/nvdiffrast/
+
+# kiuikit
+pip install git+https://github.com/ashawkey/kiuikit
+
+# To use MVdream, also install:
+pip install git+https://github.com/bytedance/MVDream
+```
+
+## Generate 3D Pokemon models
+
+After building the environment.
+
+We provide a script to generate multi type & multi animal type's Pokemon models.
+
+```
+cd ../
+python generate.py
+```
+
+It will generate many 3D pokemon models in `output/` directory.
+
+And plese run the script to transform the mesh model to cloud points.
+
+```
+pip install -r requirements
+python mesh_to_points.py
+```
+
+## Camera Calibration
+
+You can find the chess pattern provided by Opencv https://github.com/opencv/opencv/blob/3.4/doc/pattern.png
+
+Please record a video and run calibration scripts
+
+```
+python camera_calibration.py my_video.py
+```
+
+it will save intristic matrix & distortion matrix.
+
+## Visualize
+
+Then you can link your webcamera. and run this script to show your Pokemon.
+
+```
+python main.py
+```
+
+Here is ourt DEMO https://youtu.be/nOnE82SKFAM
+
+

dreamgaussian

@@ -0,0 +1 @@
+Subproject commit 56f92928dbcd92aa6c5f86efd26bd3b1c253aba0

generate.py

@@ -0,0 +1,29 @@
+import os
+import shutil
+
+def mkdir(dirname):
+    if not os.path.isdir(dirname):
+        os.mkdir(dirname)
+
+if __name__ == "__main__":
+
+
+    types = ['fire', 'water', 'electric', 'rock', 'grass', 'ice', 'steel']
+    animals = ['cat', 'dog', 'rat', 'rabbit', 'dragon', 'duck', 'turtle', 'butterfly', 'monkey', 'bee', 'fox', 'flower', 'horse', 'jellyfish', 'snake', 'Tyrannosaurus', 'dinosaur', 'fish', 'whale', 'bat', 'bear', 'deer', 'pig', 'eagle', 'chicken']
+
+
+    mkdir('./output')
+    for type_str in types:
+        for animal in animals:
+            mkdir('./output/{}_{}'.format(type_str, animal))
+            for index in range(1):
+                mkdir('./output/{}_{}/{}'.format(type_str, animal, index))
+                prompt = "a {} type pokemon, it looks like a {}".format(type_str, animal)
+                print("PROMPT: ", prompt)
+
+                os.system('python dreamgaussian/main.py --config dreamgaussian/configs/text_mv.yaml prompt="{}" save_path=output'.format(prompt))
+                os.system('python dreamgaussian/main2.py --config dreamgaussian/configs/text_mv.yaml prompt="{}" save_path=output'.format(prompt))
+
+                for file in os.listdir('./logs'):
+                    shutil.move('./logs/{}'.format(file), './output/{}_{}/{}/{}'.format(type_str, animal, index, file))
+

main.py

@@ -8,8 +8,11 @@ import random
 PAPER_WCS_POINT = np.array([[0, 0, 0], [18.5, 0, 0], [0, 26, 0], [18.5, 26, 0]], dtype=np.float32)
 CAMERA_MATRIX = np.load('camera_parameters.npy', allow_pickle=True).item()['K']
 DISTORTION_MATRIX = np.load('camera_parameters.npy', allow_pickle=True).item()['dist']
-types = ['fire', 'water', 'electric', 'rock', 'grass', 'ice', 'steel']
+# types = ['fire', 'water', 'electric', 'rock', 'grass', 'ice', 'steel']
-animals = ['cat', 'dog', 'rat', 'rabbit', 'dragon', 'duck', 'turtle', 'butterfly', 'monkey', 'bee', 'fox', 'flower', 'horse', 'jellyfish', 'snake', 'Tyrannosaurus', 'dinosaur', 'fish', 'whale', 'bat', 'bear', 'deer', 'pig', 'eagle', 'chicken']
+# animals = ['cat', 'dog', 'rat', 'rabbit', 'dragon', 'duck', 'turtle', 'butterfly', 'monkey', 'bee', 'fox', 'flower', 'horse', 'jellyfish', 'snake', 'Tyrannosaurus', 'dinosaur', 'fish', 'whale', 'bat', 'bear', 'deer', 'pig', 'eagle', 'chicken']
+
+types = ['fire']
+animals = ['cat']
 
 def get_new_approx(approx, previous_approx):
     def distance(point, pre_point):
@@ -127,7 +130,7 @@ def visualization(inverse_matrix_M, pokemon):
     pokemon_point = pokemon.get_position()[:, :3]
     pokemon_point[:, 2] = -pokemon_point[:, 2]
     pcd.points = o3d.utility.Vector3dVector(pokemon_point)
-    pcd.colors = o3d.utility.Vector3dVector(pokemon.get_colors())
+    pcd.colors = o3d.utility.Vector3dVector(pokemon.get_colors()[:, ::-1])
     vis.add_geometry(pcd)
 
 
@@ -218,6 +221,15 @@ def render_pokemon(frame, M, camera_position, pokemon):
         point2D = np.int_(point2D[::-1])
         if 0 <= point2D[0] and point2D[0] < frame.shape[1] and 0 <= point2D[1] and point2D[1] < frame.shape[0]:
             frame = cv2.circle(frame, tuple(point2D), 10,color.astype(int).tolist(), thickness=-1)
+    frame = cv2.putText(
+        frame,
+        "{} type, {} pokemon".format(pokemon.type, pokemon.animal),
+        (350, 50),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        2,
+        (255, 255, 0),
+        2
+    )
     return frame
 
 
@@ -270,15 +282,14 @@ if __name__ == '__main__':
                 if previous_approx == []:
                     print("INITIAL")
                     previous_approx = initial_approx(approx)
-                    print(previous_approx)
                 new_approx = get_new_approx(approx, previous_approx)
                 previous_approx = new_approx
 
                 paper_ccs_point = np.concatenate(new_approx, axis=0, dtype=np.float32)
 
                 # 畫邊緣 & 四點
-                plot_corner_points(frame, new_approx)
+                # plot_corner_points(frame, new_approx)
-                cv2.drawContours(frame, [approx[:, :, ::-1]], -1, (0, 255, 0), 2)  # 繪製輪廓
+                # cv2.drawContours(frame, [approx[:, :, ::-1]], -1, (0, 255, 0), 2)  # 繪製輪廓
 
                 # 算 rotaion & translation
                 success, rotation_vector, translation_vector = cv2.solvePnP(PAPER_WCS_POINT, paper_ccs_point, \
@@ -315,7 +326,7 @@ if __name__ == '__main__':
             cv2.imshow('Paper Detection', frame)
 
             # 按下 'q' 鍵退出迴圈
-            key = cv2.waitKey(33) & 0xFF
+            key = cv2.waitKey(17) & 0xFF
             if key == ord('q'):  # 等待 33ms (1秒 = 1000ms, 1秒顯示幀)
                 break
             if key == ord(' '):
@@ -340,6 +351,7 @@ if __name__ == '__main__':
 
 
     #  畫 camera pose
+    '''
     M_inv = []
     for index in range(len(rotation_vectors)):
         R = rotation_vectors[index]
@@ -350,6 +362,7 @@ if __name__ == '__main__':
 
     # load Pokemon points
     visualization(M_inv, pokemon)
+    '''
 
 
     

models/fire_cat/0/output.mtl

@@ -0,0 +1,8 @@
+newmtl defaultMat 
+Ka 1 1 1 
+Kd 1 1 1 
+Ks 0 0 0 
+Tr 1 
+illum 1 
+Ns 0 
+map_Kd output_albedo.png 

models/fire_cat/0/output_mesh.mtl

@@ -0,0 +1,8 @@
+newmtl defaultMat 
+Ka 1 1 1 
+Kd 1 1 1 
+Ks 0 0 0 
+Tr 1 
+illum 1 
+Ns 0 
+map_Kd output_mesh_albedo.png 

requirements.txt

@@ -0,0 +1,7 @@
+numpy
+trimesh
+matplotlib
+open3d
+opencv-python
+pillow
+