Everyday Postgres: Specifying all your INSERT columns

Postgres has so many convenient features, including the ability to not provide a list of columns to an INSERT.

For example:

CREATE TABLE temp_product_versions ( LIKE product_versions );
INSERT INTO temp_product_versions ( SELECT * from product_versions ); 

That’s pretty badass.

However, you may encounter trouble in paradise later if you use this kind of shortcut in production code.

See if you can spot the error in this code sample below.

Here’s the error message:

ERROR:  column "is_rapid_beta" is of type boolean but expression is of type citext
LINE 10:     repository
             ^
HINT:  You will need to rewrite or cast the expression.
QUERY:  INSERT INTO releases_recent
SELECT 'MetroFirefox',
    version,
    beta_number,
    build_id
    update_channel,
    platform,
    is_rapid,
    is_rapid_beta,
    repository
FROM releases_recent
    JOIN products
        ON products.product_name = 'MetroFirefox'
WHERE releases_recent.product_name = 'Firefox'
    AND major_version_sort(releases_recent.version)
        >= major_version_sort(products.rapid_release_version)
CONTEXT:  PL/pgSQL function update_product_versions(integer) line 102 at SQL statement

And here’s the code (long!)

CREATE OR REPLACE FUNCTION update_product_versions(
product_window integer DEFAULT 30
)
RETURNS boolean
LANGUAGE plpgsql
SET work_mem TO '512MB'
SET maintenance_work_mem TO '512MB'
AS $$
BEGIN
-- daily batch update function for new products and versions
-- reads data from releases_raw, cleans it
-- and puts data corresponding to the new versions into
-- product_versions and related tables
-- is cumulative and can be run repeatedly without issues
-- * covers FennecAndroid and ESR releases
-- * now only compares releases from the last 30 days
-- * restricts to only the canonical "repositories"
-- * covers webRT
-- * covers rapid betas, but no more final betas
-- create temporary table, required because
-- all of the special cases
create temporary table releases_recent
ON commit drop
AS
select COALESCE ( specials.product_name, products.product_name )
AS product_name,
releases_raw.version,
releases_raw.beta_number,
releases_raw.build_id,
releases_raw.update_channel,
releases_raw.platform,
(major_version_sort(version) >= major_version_sort(rapid_release_version))
AS is_rapid,
is_rapid_beta(releases_raw.update_channel, version, rapid_beta_version::major_version)
AS is_rapid_beta,
releases_raw.repository
FROM releases_raw
JOIN products ON releases_raw.product_name = products.release_name
JOIN release_repositories
ON releases_raw.repository = release_repositories.repository
LEFT OUTER JOIN special_product_platforms AS specials
ON releases_raw.platform::citext = specials.platform
AND releases_raw.product_name = specials.release_name
AND releases_raw.repository = specials.repository
AND releases_raw.update_channel = specials.release_channel
AND major_version_sort(version) >= major_version_sort(min_version)
WHERE
build_date(build_id) > (current_date - product_window)
AND version_matches_channel(releases_raw.version,
releases_raw.update_channel::citext);
-- fix ESR versions
UPDATE releases_recent
SET update_channel = 'esr'
WHERE update_channel ILIKE 'release'
AND version ILIKE '%esr';
-- insert WebRT "releases", which are copies of Firefox releases
-- insert them only if the FF release is greater than the first
-- release for WebRT
INSERT INTO releases_recent
SELECT 'WebappRuntime',
version,
beta_number,
build_id,
update_channel,
platform,
is_rapid,
is_rapid_beta,
repository
FROM releases_recent
JOIN products
ON products.product_name = 'WebappRuntime'
WHERE releases_recent.product_name = 'Firefox'
AND major_version_sort(releases_recent.version)
>= major_version_sort(products.rapid_release_version);
-- insert WebRTmobile "releases", which are copies of Fennec releases
-- insert them only if the Fennec release is greater than the first
-- release for WebRTmobile
INSERT INTO releases_recent
SELECT 'WebappRuntimeMobile',
version,
beta_number,
build_id,
update_channel,
platform,
is_rapid,
is_rapid_beta,
repository
FROM releases_recent
JOIN products
ON products.product_name = 'WebappRuntimeMobile'
WHERE releases_recent.product_name = 'Fennec'
AND major_version_sort(releases_recent.version)
>= major_version_sort(products.rapid_release_version);
-- insert MetroFirefox "releases", which are copies of Firefox releases
-- insert them only if the FF release is greater than the first
-- release for WebRT
INSERT INTO releases_recent
SELECT 'MetroFirefox',
version,
beta_number,
build_id
update_channel,
platform,
is_rapid,
is_rapid_beta,
repository
FROM releases_recent
JOIN products
ON products.product_name = 'MetroFirefox'
WHERE releases_recent.product_name = 'Firefox'
AND major_version_sort(releases_recent.version)
>= major_version_sort(products.rapid_release_version);
-- now put it in product_versions
-- first releases, aurora and nightly and non-rapid betas
insert into product_versions (
product_name,
major_version,
release_version,
version_string,
beta_number,
version_sort,
build_date,
sunset_date,
build_type,
has_builds,
build_type_enum
)
select releases_recent.product_name,
to_major_version(version),
version,
version_string(version, releases_recent.beta_number),
releases_recent.beta_number,
version_sort(version, releases_recent.beta_number),
build_date(min(build_id)),
sunset_date(min(build_id), releases_recent.update_channel),
releases_recent.update_channel::citext,
(releases_recent.update_channel IN ('aurora', 'nightly')),
releases_recent.update_channel::build_type_enum as build_type_enum
from releases_recent
left outer join product_versions ON
( releases_recent.product_name = product_versions.product_name
AND releases_recent.version = product_versions.release_version
AND releases_recent.beta_number IS NOT DISTINCT FROM product_versions.beta_number )
where is_rapid
AND product_versions.product_name IS NULL
AND NOT releases_recent.is_rapid_beta
group by releases_recent.product_name, version,
releases_recent.beta_number,
releases_recent.update_channel::citext, releases_recent.update_channel;
-- insert rapid betas "parent" products
-- these will have a product, but no builds
insert into product_versions (
product_name,
major_version,
release_version,
version_string,
beta_number,
version_sort,
build_date,
sunset_date,
build_type,
is_rapid_beta,
has_builds,
build_type_enum
)
select products.product_name,
to_major_version(version),
version,
version || 'b',
0,
version_sort(version, 0),
build_date(min(build_id)),
sunset_date(min(build_id), 'beta' ),
'beta',
TRUE,
TRUE,
'beta'
from releases_recent
join products ON releases_recent.product_name = products.release_name
left outer join product_versions ON
( releases_recent.product_name = product_versions.product_name
AND releases_recent.version = product_versions.release_version
AND product_versions.beta_number = 0 )
where is_rapid
and releases_recent.is_rapid_beta
and product_versions.product_name IS NULL
group by products.product_name, version;
-- finally, add individual betas for rapid_betas
-- these need to get linked to their master rapid_beta
insert into product_versions (
product_name,
major_version,
release_version,
version_string,
beta_number,
version_sort,
build_date,
sunset_date,
build_type,
rapid_beta_id,
build_type_enum
)
select products.product_name,
to_major_version(version),
version,
version_string(version, releases_recent.beta_number),
releases_recent.beta_number,
version_sort(version, releases_recent.beta_number),
build_date(min(build_id)),
rapid_parent.sunset_date,
'beta',
rapid_parent.product_version_id,
'beta'
from releases_recent
join products ON releases_recent.product_name = products.release_name
left outer join product_versions ON
( releases_recent.product_name = product_versions.product_name
AND releases_recent.version = product_versions.release_version
AND product_versions.beta_number = releases_recent.beta_number )
join product_versions as rapid_parent ON
releases_recent.version = rapid_parent.release_version
and releases_recent.product_name = rapid_parent.product_name
and rapid_parent.is_rapid_beta
where is_rapid
and releases_recent.is_rapid_beta
and product_versions.product_name IS NULL
group by products.product_name, version, rapid_parent.product_version_id,
releases_recent.beta_number, rapid_parent.sunset_date;
-- add build ids
-- note that rapid beta parent records will have no buildids of their own
insert into product_version_builds
(product_version_id, build_id, platform, repository)
select distinct product_versions.product_version_id,
releases_recent.build_id,
releases_recent.platform,
releases_recent.repository
from releases_recent
join product_versions
ON releases_recent.product_name = product_versions.product_name
AND releases_recent.version = product_versions.release_version
AND releases_recent.update_channel = product_versions.build_type
AND ( releases_recent.beta_number IS NOT DISTINCT FROM product_versions.beta_number )
left outer join product_version_builds ON
product_versions.product_version_id = product_version_builds.product_version_id
AND releases_recent.build_id = product_version_builds.build_id
AND releases_recent.platform = product_version_builds.platform
where product_version_builds.product_version_id is null;
drop table releases_recent;
RETURN TRUE;
END;
$$;

I’m sure quite a few of you found the problem right away. For the rest of us…

Here’s the error message you get if you specify the columns for the INSERT:

ERROR:  INSERT has more target columns than expressions
LINE 10:     repository
             ^
QUERY:  INSERT INTO releases_recent (
    product_name,
    version,
    beta_number,
    build_id,
    update_channel,
    platform,
    is_rapid,
    is_rapid_beta,
    repository
)
SELECT 'MetroFirefox',
    version,
    beta_number,
    build_id
    update_channel,
    platform,
    is_rapid,
    is_rapid_beta,
    repository
FROM releases_recent
    JOIN products
        ON products.product_name = 'MetroFirefox'
WHERE releases_recent.product_name = 'Firefox'
    AND major_version_sort(releases_recent.version)
        >= major_version_sort(products.rapid_release_version)
CONTEXT:  PL/pgSQL function update_product_versions(integer) line 112 at SQL statement

Now, it should be completely obvious. There’s a missing comma after build_id.

Implicit columns for INSERT are a convenient feature when you’re getting work done quickly, they are definitely not a best practice when writing production code. If you know of a linting tool for plpgsql that calls this kind of thing out, I’d love to hear about it and use it.