Jeff Taylor
Principal Data Consultant
Database Consulting, LLC
SQL Server
Partitioning
Jeff Taylor
Principal Data Consultant
Database Consulting, LLC
Super Hero Suit
Except for May, November and December
Jeff Taylor
Principal Data Consultant
Database Consulting, LLC
SQL Server
Partitioning
USE [StackOverflow]
GO
SELECT
YEAR(v.CreationDate) AS YearValue,
COUNT(*) AS Records
FROM
dbo.Votes AS v
GROUP BY
YEAR(v.CreationDate)
ORDER BY
1;
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_Empty]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2008]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2009]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2010]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2011]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2012]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2013]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2014]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2015]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2016]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2017]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2018]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2019]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2020]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2021]
GO
ALTER DATABASE [StackOverflow] ADD FILEGROUP [PG_Yearly_2022]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_Empty',
FILENAME = N'C:\Files\Databases\2025\StackOverflow_PG_Yearly_Empty.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_Empty]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2008',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2008.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2008]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2009',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2009.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2009]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2010',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2010.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2010]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2011',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2011.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2011]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2012',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2012.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2012]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2013',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2013.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2013]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2014',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2014.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2014]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2015',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2015.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2015]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2016',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2016.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2016]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2017',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2017.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2017]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2018',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2018.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2018]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2019',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2019.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2019]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2020',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2020.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2020]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2021',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2021.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2021]
GO
ALTER DATABASE [StackOverflow] ADD FILE (
NAME = N'StackOverflow_PG_Yearly_2022',
FILENAME = N'C:\Files\Databases\StackOverflow_PG_Yearly_2022.ndf' ,
SIZE = 1MB, FILEGROWTH = 128MB)
TO FILEGROUP [PG_Yearly_2022]
GO
CREATE PARTITION FUNCTION [PF_Yearly] (DATE)
AS RANGE RIGHT FOR VALUES (
'2008-01-01',
'2009-01-01',
'2010-01-01',
'2011-01-01',
'2012-01-01',
'2013-01-01',
'2014-01-01',
'2015-01-01',
'2016-01-01',
'2017-01-01',
'2018-01-01',
'2019-01-01',
'2020-01-01',
'2021-01-01',
'2022-01-01');
CREATE PARTITION SCHEME [PS_Yearly]
AS PARTITION [PF_Yearly] TO (
[PG_Yearly_Empty],
[PG_Yearly_2008],
[PG_Yearly_2009],
[PG_Yearly_2010],
[PG_Yearly_2011],
[PG_Yearly_2012],
[PG_Yearly_2013],
[PG_Yearly_2014],
[PG_Yearly_2015],
[PG_Yearly_2016],
[PG_Yearly_2017],
[PG_Yearly_2018],
[PG_Yearly_2019],
[PG_Yearly_2020],
[PG_Yearly_2021],
[PG_Yearly_2022])
GO
CREATE TABLE [dbo].[Votes_Partitioned](
[Id] [INT] NOT NULL,
[PostId] [INT] NOT NULL,
[UserId] [INT] NULL,
[BountyAmount] [INT] NULL,
[VoteTypeId] [INT] NOT NULL,
[CreationDate] DATE NOT NULL,
CONSTRAINT [PK_Votes_Partitioned] PRIMARY KEY CLUSTERED
(
[Id] ASC,
[CreationDate] ASC
) WITH (DATA_COMPRESSION = PAGE)
ON [PS_Yearly]([CreationDate])) ON [PS_Yearly]([CreationDate])
GO
INSERT INTO
dbo.Votes_Partitioned WITH (TABLOCK)
(Id, PostId, UserId, BountyAmount, VoteTypeId, CreationDate)
SELECT
Id,
PostId,
UserId,
BountyAmount,
VoteTypeId,
CreationDate
FROM
dbo.Votes;
--9 minutes 11 seconds --231,441,846
--19 seconds
CREATE NONCLUSTERED INDEX [IX_Votes_Partitioned_UserId_BountyAmount_VoteTypeId_CreationDate]
ON [dbo].[Votes_Partitioned]
(
UserId,
BountyAmount,
VoteTypeId,
CreationDate
)
WHERE BountyAmount IS NOT NULL
WITH (SORT_IN_TEMPDB = ON, DATA_COMPRESSION = PAGE) ON [PS_Yearly]([CreationDate])
GO
SELECT
p.partition_number,
ps.name AS PartitionScheme,
pf.name AS PartitionFunction,
prv.value AS BoundaryValue,
SUM(p.rows) AS RowCounts
FROM
sys.tables t
INNER JOIN sys.indexes i
ON t.object_id = i.object_id AND i.index_id IN (0, 1) -- Heap or clustered index
INNER JOIN sys.partitions p
ON i.object_id = p.object_id AND i.index_id = p.index_id
INNER JOIN sys.partition_schemes ps
ON i.data_space_id = ps.data_space_id
INNER JOIN sys.partition_functions pf
ON ps.function_id = pf.function_id
LEFT OUTER JOIN sys.partition_range_values prv
ON pf.function_id = prv.function_id AND p.partition_number = prv.boundary_id + 1
WHERE
t.name = 'Votes_Partitioned' AND SCHEMA_NAME(t.schema_id) = 'dbo'
AND pf.name = 'PF_Yearly'
GROUP BY
p.partition_number, ps.name, pf.name, prv.value
ORDER BY
p.partition_number;
SELECT
TOP 100
COUNT(v.PostId) AS Posts,
COUNT(v.UserId) AS Users,
SUM(v.BountyAmount) AS BountyAmount,
v.VoteTypeId,
YEAR(v.CreationDate) AS YearValue
FROM
dbo.Votes_Partitioned AS v
WHERE
v.BountyAmount IS NOT NULL
AND $PARTITION.PF_Yearly(CreationDate) =
(
SELECT
MAX($PARTITION.PF_Yearly(CreationDate))
FROM
dbo.Votes_Partitioned
)
GROUP BY
v.VoteTypeId,
YEAR(v.CreationDate)
ORDER BY
5 DESC;
SELECT
COUNT(v.PostId) AS Posts,
COUNT(v.UserId) AS Users,
SUM(v.BountyAmount) AS BountyAmount,
v.VoteTypeId,
YEAR(v.CreationDate) AS YearValue
FROM
dbo.Votes_Partitioned AS v
WHERE
v.BountyAmount IS NOT NULL
AND $PARTITION.PF_Yearly(CreationDate) =
(
SELECT
MAX($PARTITION.PF_Yearly(CreationDate))
FROM
dbo.Votes_Partitioned
)
GROUP BY
v.VoteTypeId,
YEAR(v.CreationDate)
ORDER BY
5 DESC
OFFSET 0 ROWS FETCH NEXT 100 ROWS ONLY;
--38 minutes 39 seconds
BACKUP DATABASE [StackOverflow]
TO
DISK = N'C:\Files\DatabaseBackups\2025\StackOverflow2025.bak'
WITH
NOFORMAT,
NOINIT,
NAME = N'StackOverflow-Full Database Backup',
SKIP,
NOREWIND,
NOUNLOAD,
STATS = 10,
CHECKSUM;
GO
DECLARE @backupSetId AS INT;
SELECT
@backupSetId = position
FROM
msdb..backupset
WHERE
database_name = N'StackOverflow'
AND backup_set_id =
(
SELECT
MAX(backup_set_id)
FROM
msdb..backupset
WHERE
database_name = N'StackOverflow'
);
IF @backupSetId IS NULL
BEGIN
RAISERROR(N'Verify failed. Backup information for database ''StackOverflow'' not found.', 16, 1);
END;
RESTORE VERIFYONLY
FROM
DISK = N'C:\Files\DatabaseBackups\2025\StackOverflow2025.bak'
WITH
FILE = @backupSetId,
NOUNLOAD,
NOREWIND;
GO
--37 minute 20 seconds
BACKUP DATABASE [StackOverflow]
TO
DISK = N'C:\Files\DatabaseBackups\2025\StackOverflow2025.Compression.bak'
WITH
NOFORMAT,
NOINIT,
NAME = N'StackOverflow-Full Database Backup',
SKIP,
NOREWIND,
NOUNLOAD,
COMPRESSION,
STATS = 10,
CHECKSUM;
GO
DECLARE @backupSetId AS INT;
SELECT
@backupSetId = position
FROM
msdb..backupset
WHERE
database_name = N'StackOverflow'
AND backup_set_id =
(
SELECT
MAX(backup_set_id)
FROM
msdb..backupset
WHERE
database_name = N'StackOverflow'
);
IF @backupSetId IS NULL
BEGIN
RAISERROR(N'Verify failed. Backup information for database ''StackOverflow'' not found.', 16, 1);
END;
RESTORE VERIFYONLY
FROM
DISK = N'C:\Files\DatabaseBackups\2025\StackOverflow2025.Compression.bak'
WITH
FILE = @backupSetId,
NOUNLOAD,
NOREWIND;
GO
--1 hour 6 minutes 19 seconds
--It went to 149GB and then down to 105GB
BACKUP DATABASE [StackOverflow]
TO
DISK = N'C:\Files\DatabaseBackups\2025\StackOverflow2025.ZSTD.bak'
WITH
NOFORMAT,
NOINIT,
NAME = N'StackOverflow-Full Database Backup',
SKIP,
NOREWIND,
NOUNLOAD,
COMPRESSION (ALGORITHM = ZSTD, LEVEL = HIGH),
STATS = 10,
CHECKSUM;
GO
DECLARE @backupSetId AS INT;
SELECT
@backupSetId = position
FROM
msdb..backupset
WHERE
database_name = N'StackOverflow'
AND backup_set_id =
(
SELECT
MAX(backup_set_id)
FROM
msdb..backupset
WHERE
database_name = N'StackOverflow'
);
IF @backupSetId IS NULL
BEGIN
RAISERROR(N'Verify failed. Backup information for database ''StackOverflow'' not found.', 16, 1);
END;
RESTORE VERIFYONLY
FROM
DISK = N'C:\Files\DatabaseBackups\2025\StackOverflow2025.ZSTD.bak'
WITH
FILE = @backupSetId,
NOUNLOAD,
NOREWIND;
GO
DECLARE @FileGroupName NVARCHAR(128), @SQL NVARCHAR(MAX);
-- Get the latest filegroup name
SELECT TOP 1 @FileGroupName = name FROM sys.filegroups ORDER BY data_space_id DESC;
--Create Backup command
SET @SQL = '
BACKUP DATABASE StackOverflow
FILEGROUP = ''' + @FileGroupName + '''
TO DISK = ''C:\Files\DatabaseBackups\2025\StackOverflow_' + @FileGroupName + '.bak''
WITH NOFORMAT,
NOINIT,
NAME = N''StackOverflow-' + @FileGroupName + ' Database Backup'',
SKIP,
NOREWIND,
NOUNLOAD,
COMPRESSION (ALGORITHM = ZSTD, LEVEL = HIGH),
STATS = 10,
CHECKSUM;';
-- Execute the backup
EXEC sp_executesql @SQL;
SELECT
name,
is_incremental,
object_id
FROM
sys.stats
WHERE
object_id = OBJECT_ID('dbo.Votes_Partitioned');
CREATE STATISTICS Votes_Partitioned_LatestPartition
ON dbo.Votes_Partitioned
(
CreationDate
)
WITH
FULLSCAN,
INCREMENTAL = ON;
DECLARE @MaxPartition INT, @SQLQuery NVARCHAR(MAX);
SELECT
@MaxPartition = MAX(p.partition_number)
FROM
sys.partitions p
JOIN
sys.tables t ON p.object_id = t.object_id
WHERE
t.name = 'Votes_Partitioned';
SET @SQLQuery = 'UPDATE STATISTICS dbo.Votes_Partitioned Votes_Partitioned_LatestPartition
WITH
RESAMPLE ON PARTITIONS (' + CAST(@MaxPartition AS VARCHAR(7)) + ');'
EXEC sp_executesql @SQLQuery
DECLARE @MaxPartition INT;
SELECT
@MaxPartition = MAX(p.partition_number)
FROM
sys.partitions p
JOIN
sys.tables t ON p.object_id = t.object_id
WHERE
t.name = 'Votes_Partitioned';
ALTER INDEX ALL ON dbo.Votes_Partitioned REBUILD PARTITION = @MaxPartition;
SELECT
p.partition_number,
ps.name AS PartitionScheme,
pf.name AS PartitionFunction,
prv.value AS BoundaryValue,
SUM(p.rows) AS RowCounts
FROM
sys.tables t
INNER JOIN sys.indexes i
ON t.object_id = i.object_id AND i.index_id IN (0, 1) -- Heap or clustered index
INNER JOIN sys.partitions p
ON i.object_id = p.object_id AND i.index_id = p.index_id
INNER JOIN sys.partition_schemes ps
ON i.data_space_id = ps.data_space_id
INNER JOIN sys.partition_functions pf
ON ps.function_id = pf.function_id
LEFT OUTER JOIN sys.partition_range_values prv
ON pf.function_id = prv.function_id AND p.partition_number = prv.boundary_id + 1
WHERE
t.name = 'Votes_Partitioned' AND SCHEMA_NAME(t.schema_id) = 'dbo'
AND pf.name = 'PF_Yearly'
GROUP BY
p.partition_number, ps.name, pf.name, prv.value
ORDER BY
p.partition_number;
CREATE TABLE [dbo].[Votes_History](
[Id] [int] NOT NULL,
[PostId] [int] NOT NULL,
[UserId] [int] NULL,
[BountyAmount] [int] NULL,
[VoteTypeId] [int] NOT NULL,
[CreationDate] [date] NOT NULL,
CONSTRAINT [PK_Votes_History] PRIMARY KEY CLUSTERED
(
[Id] ASC, [CreationDate] ASC
)WITH (DATA_COMPRESSION = PAGE) ON [PG_Yearly_2008]
) ON [PG_Yearly_2008]
GO
CREATE NONCLUSTERED INDEX [IX_Votes_Partitioned_UserId_BountyAmount_VoteTypeId_CreationDate] ON [dbo].[Votes_History]
(
[UserId] ASC, [BountyAmount] ASC, [VoteTypeId] ASC, [CreationDate] ASC
)
WHERE ([BountyAmount] IS NOT NULL)
WITH (DATA_COMPRESSION = PAGE) ON [PG_Yearly_2008]
GO
ALTER TABLE dbo.Votes_History ADD CONSTRAINT CK_Votes_History_Range
CHECK (CreationDate >= '2008-01-01' AND CreationDate < '2009-01-01');
DECLARE @PartitionNumber INT = 2;
ALTER TABLE dbo.Votes_Partitioned SWITCH PARTITION @PartitionNumber TO dbo.Votes_History;
SELECT MIN(CreationDate) AS MinDate, MAX(CreationDate) AS MaxDate FROM dbo.Votes_History;
ALTER TABLE dbo.Votes_History SWITCH TO dbo.Votes_Partitioned PARTITION 2;
Thank you for attending my session today.
If you have any additional questions, please don't hesitate to reach out.