OpenCV4入门164：超分辨率

索引地址：系列索引

超分辨率（Super Resolution）指的是分辨率较高的比如4K、8K等等，但是一般这个做动词，指将低分辨率的视频图片通过某种方法变为高分辨率的行为，考虑到原始数据的状况，一般就是720P/1080P这样。

泰坦尼克号、终结者2这样的电影有过高清修复重映，一般就是将分辨率放大提高画质。这些都是胶片拍摄的。像一些电影、动漫等等，我们不是制作方没有原始数据，那么就只能通过软件的方式提升画质。

下面介绍OpenCV提供的超分辨率方法，实际在使用时一般使用神经网络+深度学习方法。比如

  • waifu2x
  • SRMD
  • Anime4k
  • Realsr
  • …

一些工具比如SFVI等工具就是整合上面的一些工具。

测试

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
#include <iostream>
#include <iomanip>
#include <string>

#include "opencv2/core.hpp"
#include "opencv2/core/utility.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/superres.hpp"
#include "opencv2/superres/optical_flow.hpp"
#include "opencv2/opencv_modules.hpp"

using namespace std;
using namespace cv;
using namespace cv::superres;

static Ptr<DenseOpticalFlowExt> createOptFlow(string name);

int main(int argc, char *argv[])
{
    // 1-Initialize the initial parameters

    // Input and output video
    string inputVideoName;
    string outputVideoName;
    if (argc > 1)
        inputVideoName = argv[1];
    else
        inputVideoName = ".\\tree.avi";

    if (argc > 2)
        outputVideoName = argv[2];
    else
        outputVideoName = ".\\tree_superresolution.avi";

    // Scale factor
    const int scale = 4;

    // Iterations count
    const int iterations = 180;

    // Radius of the temporal search area
    const int temporalAreaRadius =4;

    // Optical flow algorithm
    string optFlow = "farneback";
       // optFlow = "farneback";
       // optFlow = "tvl1";
       // optFlow = "brox";
       // optFlow = "pyrlk";

    // Playback speed ouput
    double outputFps = 25.0;

    // 2- Create an optical flow method
    Ptr<DenseOpticalFlowExt> optical_flow = createOptFlow(optFlow);

    if (optical_flow.empty())
        return -1;

    // 3- Create the superresolution method and set its parameters
    Ptr<SuperResolution> superRes;
    superRes = createSuperResolution_BTVL1();

    superRes->setOpticalFlow(optical_flow);
    superRes->setScale(scale);
    superRes->setIterations(iterations);
    superRes->setTemporalAreaRadius(temporalAreaRadius);

    Ptr<FrameSource> frameSource;
    frameSource = createFrameSource_Video(inputVideoName);

    superRes->setInput(frameSource);

    // Not use the first frame
    Mat frame;
    frameSource->nextFrame(frame);

    // 4- Processing the input video with the superresolution
    // Show the initial options
    cout << "Input                  : " << inputVideoName << " " << frame.size() << endl;
    cout << "Output                 : " << outputVideoName << endl;
    cout << "Playback speed output  : " << outputFps << endl;
    cout << "Scale factor           : " << scale << endl;
    cout << "Iterations             : " << iterations << endl;
    cout << "Temporal radius        : " << temporalAreaRadius << endl;
    cout << "Optical Flow           : " << optFlow << endl;
    cout << endl;

    VideoWriter writer;
    double start_time,finish_time;

    for (int i = 0;; ++i)
    {
        cout << '[' << setw(3) << i << "] : ";
        Mat result;

        // Calculate the processing time
        start_time = getTickCount();
        superRes->nextFrame(result);
        finish_time = getTickCount();
        cout << (finish_time - start_time)/getTickFrequency() << " secs, Size: " << result.size() << endl;

        if (result.empty())
            break;

        // Show the result
        imshow("Super Resolution", result);

        if (waitKey(1000) > 0)
            break;

        // Save the result on output file
        if (!outputVideoName.empty())
        {
            if (!writer.isOpened())
                writer.open(outputVideoName, VideoWriter::fourcc('X', 'V', 'I', 'D'), outputFps, result.size());
            writer << result;
        }
    }
    writer.release();

    return 0;
}

static Ptr<DenseOpticalFlowExt> createOptFlow(string name)
{
    if (name == "farneback")
       return createOptFlow_Farneback();

    else if (name == "tvl1")
        return createOptFlow_DualTVL1();

    else if (name == "brox")
        return createOptFlow_Brox_CUDA();

    else if (name == "pyrlk")
        return createOptFlow_PyrLK_CUDA();

    else
        cerr << "Incorrect Optical Flow algorithm - " << name << endl;

    return Ptr<DenseOpticalFlowExt>();
}

用来测试的视频信息为

1
2
3
4
5
$ ffprobe test.avi                          
...
Input #0, avi, from 'test.avi':
  Duration: 00:00:16.53, start: 0.000000, bitrate: 1292 kb/s
  Stream #0:0: Video: msmpeg4v2 (MP42 / 0x3234504D), yuv420p, 640x360, 1288 kb/s, 30 fps, 30 tbr, 30 tbn

原始视频为360p，30帧，时长为16秒

使用farneback模式进行放大，测试环境为戴尔G15 1511 i7-11800H 8核16线程，16G内存

测试程序输出为

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
$ ./sr test.avi tsr.avi  
Input                  : test.avi [640 x 360]
Output                 : tsr.avi
Playback speed output  : 25
Scale factor           : 4
Iterations             : 180
Temporal radius        : 4
Optical Flow           : farneback

[  0] : 543.328 secs, Size: [2546 x 1426]
[  1] : 86.5821 secs, Size: [2546 x 1426]
[  2] : 89.9619 secs, Size: [2546 x 1426]
[  3] : 89.7291 secs, Size: [2546 x 1426]
[  4] : 89.9414 secs, Size: [2546 x 1426]
[  5] : 90.3375 secs, Size: [2546 x 1426]
[  6] : 91.1083 secs, Size: [2546 x 1426]
[  7] : 91.6142 secs, Size: [2546 x 1426]
[  8] : 91.4761 secs, Size: [2546 x 1426]
[  9] : 91.6939 secs, Size: [2546 x 1426]
[ 10] : 91.86 secs, Size: [2546 x 1426]
[ 11] : 91.8984 secs, Size: [2546 x 1426]
[ 12] : 91.7052 secs, Size: [2546 x 1426]
[ 13] : 91.7788 secs, Size: [2546 x 1426]
[ 14] : 91.6301 secs, Size: [2546 x 1426]
[ 15] : 91.5122 secs, Size: [2546 x 1426]
[ 16] : 90.6578 secs, Size: [2546 x 1426]
[ 17] : 90.612 secs, Size: [2546 x 1426]
[ 18] : 91.185 secs, Size: [2546 x 1426]
...
[481] : 90.6462 secs, Size: [2546 x 1426]
[482] : 90.534 secs, Size: [2546 x 1426]
[483] : 90.8866 secs, Size: [2546 x 1426]
[484] : 90.6648 secs, Size: [2546 x 1426]
[485] : 96.7889 secs, Size: [2546 x 1426]
[486] : 89.6735 secs, Size: [2546 x 1426]
[487] : 77.6115 secs, Size: [2546 x 1426]
[488] : 68.7781 secs, Size: [2546 x 1426]
[489] : 61.9018 secs, Size: [2546 x 1426]
[490] : 0.00274725 secs, Size: [2546 x 1426]
[491] : 0.00269216 secs, Size: [2546 x 1426]
[492] : 0.00278245 secs, Size: [2546 x 1426]
[493] : 0.00277431 secs, Size: [2546 x 1426]
[494] : 0.00338189 secs, Size: [2546 x 1426]
[495] : 2.182e-06 secs, Size: [0 x 0]

从2022.04.29 20:19:22到2022.04.30 09:06:01耗时8个小时，期间CPU16核全满，内存接近全满，GPU未使用。

而且画质不太好，频繁出现花屏

result

输出的结果视频信息为：

1
2
3
4
5
6
7
$ ffprobe tsr.avi 
...
Input #0, avi, from 'tsr.avi':
  Metadata:
    software        : Lavf58.76.100
  Duration: 00:00:19.80, start: 0.000000, bitrate: 19202 kb/s
  Stream #0:0: Video: mpeg4 (Simple Profile) (XVID / 0x44495658), yuv420p, 2546x1426 [SAR 1:1 DAR 1273:713], 19230 kb/s, 25 fps, 25 tbr, 25 tbn

结果为1426p，25帧，时长19秒

总结

还是更现代化的好使一点

OpenCV
OpenCV SR

本博客所有文章除特别声明外，均采用 CC BY-SA 4.0 协议 ，转载请注明出处！