rk3568制冷项目驱动开发流程汇总（只适用于部分模块CIF DVP等，自用）

&i2c0 {status =  "okay";sensor@3c{status =  "okay";compatible = "firefly,pc9202";reg = <0x3c>;wd-en-gpio = <&gpio3 23 GPIO_ACTIVE_HIGH>;};avafpga: avafpga@32 {status = "okay";compatible = "ava,fpga";reg = <0x32>;clocks = <&cru CLK_CIF_OUT>;clock-names = "xvclk";power-domains = <&power RK3568_PD_VI>;pwdn-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>;reset-gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_HIGH>;rockchip,grf = <&grf>;pinctrl-names = "default";pinctrl-0 = <&cif_clk&cif_dvp_clk&cif_dvp_bus16&cif_dvp_bus8>;rockchip,camera-module-index = <0>;rockchip,camera-module-facing = "back";rockchip,camera-module-name = "default";rockchip,camera-module-lens-name = "default";rockchip,dvp_mode = "BT1120"; //BT656 or BT1120 or BT656_TESTport {cam_para_out2: endpoint {remote-endpoint = <&dvp_in_bcam>;bus-width = <16>;pclk-sample = <1>;};};};
};&rkcif {status = "okay";
};&rkcif_mmu {status = "okay";
};&rkcif_dvp {status = "okay";port {dvp_in_bcam: endpoint {remote-endpoint = <&cam_para_out2>;bus-width = <16>;};};
};

（7）/rk356x_sdk-linux5.10/kernel/drivers/media/i2c目录下增加fpga.c，并在kconfig和makefile中增加相关模块的设置

config VIDEO_FPGAtristate "AVA FPGA sensor support"depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_APIdepends on MEDIA_CAMERA_SUPPORThelpThis is a Video4Linux2 sensor driver for the AVAFPGA camera.To compile this driver as a module, choose M here: themodule will be called fpga.

obj-y += fpga.o

完整的fpga的c代码如下，亦可见附件（开始未设置ioctl，上电v4l2抓图时，打印_avafpga_start_stream enter然后崩掉，后经查阅资料以及参考rv1126的内容，加入对应的部分，可以成功获取数据，且不会崩掉）

#include <linux/clk.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/gpio/consumer.h>
#include <linux/i2c.h>
#include <linux/module.h>
#include <linux/pm_runtime.h>
#include <linux/regulator/consumer.h>
#include <linux/sysfs.h>
#include <linux/slab.h>
#include <linux/version.h>
#include <linux/rk-camera-module.h>
#include <media/media-entity.h>
#include <media/v4l2-async.h>
#include <media/v4l2-ctrls.h>
#include <media/v4l2-subdev.h>
#include <linux/pinctrl/consumer.h>
#include <linux/rk-preisp.h>#include <media/v4l2-fwnode.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/of_graph.h>
#include <linux/of_platform.h>
#include <linux/of_gpio.h>
#include <linux/mfd/syscon.h>
#include <linux/version.h>#define DRIVER_VERSION			KERNEL_VERSION(0, 0x0, 0x01)
//#define PIX_FORMAT			MEDIA_BUS_FMT_UYVY8_2X8
#define PIX_FORMAT                    MEDIA_BUS_FMT_YUYV8_2X8
#define AVAFPGA_NAME			"avafpga"struct regval {u16 addr;u8 val;
};struct avafpga_mode {u32 width;u32 height;struct v4l2_fract max_fps;u32 hts_def;u32 vts_def;u32 exp_def;const struct regval *reg_list;
};struct avafpga {struct i2c_client	*client;struct clk		*xvclk;struct gpio_desc	*reset_gpio;struct gpio_desc	*pwdn_gpio;struct v4l2_subdev	subdev;struct media_pad	pad;struct v4l2_ctrl_handler ctrl_handler;struct v4l2_ctrl	*exposure;struct v4l2_ctrl	*anal_gain;struct v4l2_ctrl	*digi_gain;struct v4l2_ctrl	*hblank;struct v4l2_ctrl	*vblank;struct v4l2_ctrl	*test_pattern;struct mutex		mutex;bool			streaming;bool			power_on;const struct avafpga_mode *cur_mode;u32			module_index;const char		*module_facing;const char		*module_name;const char		*len_name;u32			ch_nums;//hjkstruct v4l2_ctrl	*vblank_ctrl;struct v4l2_ctrl	*hblank_ctrl;u8 is_reset;
};#define to_avafpga(sd) container_of(sd, struct avafpga, subdev)static const struct avafpga_mode supported_modes[] = {{//.width = 1920,//.height = 1080,.width = 1536,.height = 576,		.max_fps = {.numerator = 10000,.denominator = 600000,},.exp_def = 0x0100,.hts_def = 0x044c * 2,.vts_def = 0x0465,}
};static int __avafpga_start_stream(struct avafpga *avafpga)
{printk(KERN_ERR "%s enter\n", __func__);avafpga->is_reset = 1;//hjk1126printk(KERN_ERR "%s exit\n", __func__);//hjk1126return 0;
}static int __avafpga_stop_stream(struct avafpga *avafpga)
{printk(KERN_ERR "%s enter\n", __func__);return 0;
}static int avafpga_s_stream(struct v4l2_subdev *sd, int on)
{struct avafpga *avafpga = to_avafpga(sd);struct i2c_client *client = avafpga->client;int ret = 0;mutex_lock(&avafpga->mutex);on = !!on;if (on == avafpga->streaming)goto unlock_and_return;if (on) {ret = pm_runtime_get_sync(&client->dev);if (ret < 0) {pm_runtime_put_noidle(&client->dev);goto unlock_and_return;}ret = __avafpga_start_stream(avafpga);if (ret) {v4l2_err(sd, "start stream failed while write regs\n");pm_runtime_put(&client->dev);goto unlock_and_return;}} else {__avafpga_stop_stream(avafpga);pm_runtime_put(&client->dev);}avafpga->streaming = on;
unlock_and_return:mutex_unlock(&avafpga->mutex);return ret;
}#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
static int avafpga_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
{struct avafpga *avafpga = to_avafpga(sd);struct v4l2_mbus_framefmt *try_fmt =v4l2_subdev_get_try_format(sd, fh->pad, 0);const struct avafpga_mode *def_mode = &supported_modes[0];mutex_lock(&avafpga->mutex);/* Initialize try_fmt */try_fmt->width = def_mode->width;try_fmt->height = def_mode->height;try_fmt->code = PIX_FORMAT;try_fmt->field = V4L2_FIELD_NONE;mutex_unlock(&avafpga->mutex);/* No crop or compose */return 0;
}
#endifstatic int avafpga_g_mbus_config(struct v4l2_subdev *sd, unsigned int pad_id,struct v4l2_mbus_config *config)
{//config->type = V4L2_MBUS_BT656;//config->flags = V4L2_MBUS_PCLK_SAMPLE_RISING;config->type = V4L2_MBUS_BT656;config->flags = V4L2_MBUS_PCLK_SAMPLE_RISING;//config->flags = V4L2_MBUS_HSYNC_ACTIVE_LOW |//V4L2_MBUS_VSYNC_ACTIVE_LOW |//V4L2_MBUS_PCLK_SAMPLE_FALLING;//config->type = V4L2_MBUS_BT656;//config->flags = RKMODULE_CAMERA_BT656_CHANNELS |//V4L2_MBUS_PCLK_SAMPLE_RISING;//config->type = V4L2_MBUS_BT656;config->flags = V4L2_MBUS_HSYNC_ACTIVE_HIGH |V4L2_MBUS_VSYNC_ACTIVE_HIGH |V4L2_MBUS_PCLK_SAMPLE_RISING;return 0;}static int avafpga_g_frame_interval(struct v4l2_subdev *sd,struct v4l2_subdev_frame_interval *fi)
{struct avafpga *avafpga = to_avafpga(sd);const struct avafpga_mode *mode = avafpga->cur_mode;mutex_lock(&avafpga->mutex);fi->interval = mode->max_fps;mutex_unlock(&avafpga->mutex);return 0;
}static int avafpga_querystd(struct v4l2_subdev *sd, v4l2_std_id *std)
{*std = V4L2_STD_ATSC;return 0;
}static int avafpga_enum_frame_interval(struct v4l2_subdev *sd,struct v4l2_subdev_pad_config *cfg,struct v4l2_subdev_frame_interval_enum *fie)
{if (fie->index >= ARRAY_SIZE(supported_modes))return -EINVAL;//if (fie->code != PIX_FORMAT)//return -EINVAL;fie->width = supported_modes[fie->index].width;fie->height = supported_modes[fie->index].height;fie->interval = supported_modes[fie->index].max_fps;return 0;
}static int avafpga_enum_mbus_code(struct v4l2_subdev *sd,struct v4l2_subdev_pad_config *cfg,struct v4l2_subdev_mbus_code_enum *code)
{if (code->index != 0)return -EINVAL;code->code = PIX_FORMAT;return 0;
}static int avafpga_enum_frame_sizes(struct v4l2_subdev *sd,struct v4l2_subdev_pad_config *cfg,struct v4l2_subdev_frame_size_enum *fse)
{if (fse->index >= ARRAY_SIZE(supported_modes))return -EINVAL;//if (fse->code != PIX_FORMAT)//hjk1126中没有这个//return -EINVAL;//hjk1126中没有这个fse->min_width  = supported_modes[fse->index].width;fse->max_width  = supported_modes[fse->index].width;fse->max_height = supported_modes[fse->index].height;fse->min_height = supported_modes[fse->index].height;return 0;
}static int avafpga_set_fmt(struct v4l2_subdev *sd,struct v4l2_subdev_pad_config *cfg,struct v4l2_subdev_format *fmt)
{struct avafpga *avafpga = to_avafpga(sd);const struct avafpga_mode *mode;mutex_lock(&avafpga->mutex);mode = avafpga->cur_mode;fmt->format.code = PIX_FORMAT;fmt->format.width = mode->width;fmt->format.height = mode->height;fmt->format.field = V4L2_FIELD_NONE;//fmt->format.field = V4L2_FIELD_INTERLACED；if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API*v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format;
#elsemutex_unlock(&avafpga->mutex);return -ENOTTY;
#endif} else {avafpga->cur_mode = mode;}mutex_unlock(&avafpga->mutex);return 0;
}static int avafpga_get_fmt(struct v4l2_subdev *sd,struct v4l2_subdev_pad_config *cfg,struct v4l2_subdev_format *fmt)
{struct avafpga *avafpga = to_avafpga(sd);const struct avafpga_mode *mode = avafpga->cur_mode;mutex_lock(&avafpga->mutex);if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
#ifdef CONFIG_VIDEO_V4L2_SUBDEV_APIfmt->format = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
#elsemutex_unlock(&avafpga->mutex);return -ENOTTY;
#endif} else {fmt->format.width = mode->width;fmt->format.height = mode->height;fmt->format.code = PIX_FORMAT;fmt->format.field = V4L2_FIELD_NONE;}mutex_unlock(&avafpga->mutex);return 0;
}static int avafpga_s_power(struct v4l2_subdev *sd, int on)
{struct avafpga *avafpga = to_avafpga(sd);struct i2c_client *client = avafpga->client;int ret = 0;mutex_lock(&avafpga->mutex);/* If the power state is not modified - no work to do. */if (avafpga->power_on == !!on)goto unlock_and_return;if (on) {ret = pm_runtime_get_sync(&client->dev);if (ret < 0) {pm_runtime_put_noidle(&client->dev);goto unlock_and_return;}avafpga->power_on = true;} else {pm_runtime_put(&client->dev);avafpga->power_on = false;}unlock_and_return:mutex_unlock(&avafpga->mutex);return ret;
}static int __avafpga_power_on(struct avafpga *avafpga)
{return 0;
}static void __avafpga_power_off(struct avafpga *avafpga)
{return;
}static int avafpga_runtime_resume(struct device *dev)
{struct i2c_client *client = to_i2c_client(dev);struct v4l2_subdev *sd = i2c_get_clientdata(client);struct avafpga *avafpga = to_avafpga(sd);return __avafpga_power_on(avafpga);
}static int avafpga_runtime_suspend(struct device *dev)
{struct i2c_client *client = to_i2c_client(dev);struct v4l2_subdev *sd = i2c_get_clientdata(client);struct avafpga *avafpga = to_avafpga(sd);__avafpga_power_off(avafpga);return 0;
}hjk///
static __maybe_unused void
avafpga_get_bt656_module_inf(struct avafpga *avafpga,struct rkmodule_bt656_mbus_info *inf)
{memset(inf, 0, sizeof(*inf));inf->flags = RKMODULE_CAMERA_BT656_PARSE_ID_LSB;switch (avafpga->ch_nums) {case 1:inf->flags |= RKMODULE_CAMERA_BT656_CHANNEL_0;break;case 2:inf->flags |= RKMODULE_CAMERA_BT656_CHANNEL_0 |RKMODULE_CAMERA_BT656_CHANNEL_1;break;case 4:inf->flags |= RKMODULE_CAMERA_BT656_CHANNELS;break;default:inf->flags |= RKMODULE_CAMERA_BT656_CHANNELS;}
}static long avafpga_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
{long ret = 0;//struct avafpga *avafpga = to_avafpga(sd);switch (cmd) {/*case RKMODULE_GET_BT656_MBUS_INFO:avafpga_get_bt656_module_inf(avafpga,(struct rkmodule_bt656_mbus_info*)arg);break;*/default:ret = -ENOTTY;break;}return ret;
}#ifdef CONFIG_COMPAT
static long avafpga_compat_ioctl32(struct v4l2_subdev *sd,unsigned int cmd, unsigned long arg)
{//void __user *up = compat_ptr(arg);//struct rkmodule_inf *inf;//struct rkmodule_awb_cfg *cfg;//int *seq;long ret = 0;//struct rkmodule_bt656_mbus_info *bt565_inf;switch (cmd) {/*case RKMODULE_GET_BT656_MBUS_INFO:bt565_inf = kzalloc(sizeof(*bt565_inf), GFP_KERNEL);if (!bt565_inf) {ret = -ENOMEM;return ret;}ret = avafpga_ioctl(sd, cmd, bt565_inf);if (!ret) {ret = copy_to_user(up, bt565_inf, sizeof(*bt565_inf));if (ret)ret = -EFAULT;}kfree(bt565_inf);break;*/default:ret = -ENOIOCTLCMD;break;}return ret;
}
#endif#define DST_XPOS 0
#define DST_YPOS 0
#define DST_WIDTH 1536 //1920
#define DST_HEIGHT 576 //1080static int avafpga_get_selection(struct v4l2_subdev *sd,struct v4l2_subdev_pad_config *cfg,struct v4l2_subdev_selection *sel)
{if (sel->target == V4L2_SEL_TGT_CROP_BOUNDS) {sel->r.left = DST_XPOS;sel->r.top = DST_YPOS;sel->r.width = DST_WIDTH;sel->r.height = DST_HEIGHT;return 0;}return -EINVAL;
}static int avafpga_initialize_controls(struct avafpga *avafpga)
{int ret;struct v4l2_ctrl_handler *handler;const struct avafpga_mode *mode;u32 h_blank, v_blank;handler = &avafpga->ctrl_handler;mode = avafpga->cur_mode;ret = v4l2_ctrl_handler_init(handler, 2);if (ret)return ret;handler->lock = &avafpga->mutex;h_blank = mode->hts_def - mode->width;avafpga->hblank_ctrl = v4l2_ctrl_new_std(handler, NULL,V4L2_CID_HBLANK, h_blank, h_blank, 1, h_blank);v_blank = mode->vts_def - mode->height;avafpga->vblank_ctrl = v4l2_ctrl_new_std(handler, NULL,V4L2_CID_VBLANK, v_blank, v_blank, 1, v_blank);if (handler->error) {ret = handler->error;dev_err(&avafpga->client->dev,"Failed to init controls(%d)\n", ret);goto err_free_handler;}avafpga->subdev.ctrl_handler = handler;return 0;err_free_handler:v4l2_ctrl_handler_free(handler);return ret;
}
hjk//static const struct dev_pm_ops avafpga_pm_ops = {SET_RUNTIME_PM_OPS(avafpga_runtime_suspend,avafpga_runtime_resume, NULL)
};#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
static const struct v4l2_subdev_internal_ops avafpga_internal_ops = {.open = avafpga_open,
};
#endifstatic const struct v4l2_subdev_core_ops avafpga_core_ops = {.s_power = avafpga_s_power,//hjk.ioctl = avafpga_ioctl,
#ifdef CONFIG_COMPAT.compat_ioctl32 = avafpga_compat_ioctl32,
#endif//hjk
};static const struct v4l2_subdev_video_ops avafpga_video_ops = {.s_stream = avafpga_s_stream,.g_frame_interval = avafpga_g_frame_interval,.querystd = avafpga_querystd,
};static const struct v4l2_subdev_pad_ops avafpga_pad_ops = {.enum_mbus_code = avafpga_enum_mbus_code,.enum_frame_size = avafpga_enum_frame_sizes,.enum_frame_interval = avafpga_enum_frame_interval,.get_fmt = avafpga_get_fmt,.set_fmt = avafpga_set_fmt,.get_selection = avafpga_get_selection,.get_mbus_config = avafpga_g_mbus_config,
};static const struct v4l2_subdev_ops avafpga_subdev_ops = {.core	= &avafpga_core_ops,.video	= &avafpga_video_ops,.pad	= &avafpga_pad_ops,
};static int avafpga_probe(struct i2c_client *client,const struct i2c_device_id *id)
{struct device *dev = &client->dev;struct avafpga *avafpga;struct v4l2_subdev *sd;char facing[2];int ret;dev_info(dev, "driver version: %02x.%02x.%02x",DRIVER_VERSION >> 16,(DRIVER_VERSION & 0xff00) >> 8,DRIVER_VERSION & 0x00ff);avafpga = devm_kzalloc(dev, sizeof(*avafpga), GFP_KERNEL);if (!avafpga)return -ENOMEM;avafpga->client = client;avafpga->cur_mode = &supported_modes[0];avafpga->xvclk = devm_clk_get(dev, "xvclk");if (IS_ERR(avafpga->xvclk)) {dev_err(dev, "Failed to get xvclk\n");return -EINVAL;}//avafpga->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);//if (IS_ERR(avafpga->reset_gpio))//dev_warn(dev, "Failed to get reset-gpios\n");//avafpga->pwdn_gpio = devm_gpiod_get(dev, "pwdn", GPIOD_OUT_LOW);//if (IS_ERR(avafpga->pwdn_gpio))//dev_warn(dev, "Failed to get pwdn-gpios\n");mutex_init(&avafpga->mutex);sd = &avafpga->subdev;v4l2_i2c_subdev_init(sd, client, &avafpga_subdev_ops);//hjkret = avafpga_initialize_controls(avafpga);if (ret) {dev_err(dev, "Failed to initialize controls fpga\n");goto err_free_handler;}//hjkret = __avafpga_power_on(avafpga);if (ret)goto err_free_handler;#ifdef CONFIG_VIDEO_V4L2_SUBDEV_APIsd->internal_ops = &avafpga_internal_ops;sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |V4L2_SUBDEV_FL_HAS_EVENTS;
#endif#if defined(CONFIG_MEDIA_CONTROLLER)avafpga->pad.flags = MEDIA_PAD_FL_SOURCE;sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;ret = media_entity_pads_init(&sd->entity, 1, &avafpga->pad);if (ret < 0)goto err_power_off;
#endifmemset(facing, 0, sizeof(facing));if (!sd->dev)dev_err(dev, "sd dev is null\n");snprintf(sd->name, sizeof(sd->name), "m%02d_%s_%s %s",avafpga->module_index, facing,AVAFPGA_NAME, dev_name(sd->dev));ret = v4l2_async_register_subdev_sensor_common(sd);if (ret) {dev_err(dev, "v4l2 async register subdev failed\n");goto err_clean_entity;}pm_runtime_set_active(dev);pm_runtime_enable(dev);pm_runtime_idle(dev);dev_err(dev, "v4l2 async register subdev sucessfully\n");return 0;err_clean_entity:
#if defined(CONFIG_MEDIA_CONTROLLER)media_entity_cleanup(&sd->entity);
#endif
err_power_off:__avafpga_power_off(avafpga);
err_free_handler:v4l2_ctrl_handler_free(&avafpga->ctrl_handler);mutex_destroy(&avafpga->mutex);return ret;
}static int avafpga_remove(struct i2c_client *client)
{struct v4l2_subdev *sd = i2c_get_clientdata(client);struct avafpga *avafpga = to_avafpga(sd);v4l2_async_unregister_subdev(sd);
#if defined(CONFIG_MEDIA_CONTROLLER)media_entity_cleanup(&sd->entity);
#endifv4l2_ctrl_handler_free(&avafpga->ctrl_handler);mutex_destroy(&avafpga->mutex);pm_runtime_disable(&client->dev);if (!pm_runtime_status_suspended(&client->dev))__avafpga_power_off(avafpga);pm_runtime_set_suspended(&client->dev);return 0;
}#if IS_ENABLED(CONFIG_OF)
static const struct of_device_id avafpga_of_match[] = {{ .compatible = "ava,fpga" },{},
};
MODULE_DEVICE_TABLE(of, avafpga_of_match);
#endifstatic const struct i2c_device_id avafpga_match_id[] = {{"ava,fpga", 0 },{},
};static struct i2c_driver avafpga_i2c_driver = {.driver = {.name = AVAFPGA_NAME,.of_match_table = of_match_ptr(avafpga_of_match),},.probe		= &avafpga_probe,.remove		= &avafpga_remove,.id_table	= avafpga_match_id,
};static int __init sensor_mod_init(void)
{return i2c_add_driver(&avafpga_i2c_driver);
}static void __exit sensor_mod_exit(void)
{i2c_del_driver(&avafpga_i2c_driver);
}device_initcall_sync(sensor_mod_init);
module_exit(sensor_mod_exit);MODULE_DESCRIPTION("AVA FPGA sensor driver");
MODULE_LICENSE("GPL v2");

编译后会在路径下生成fpga.o，加载进内核模块

具体的调试结果如下